From 7e7548f4d9114df87543ceda2bac7325778d9d4d Mon Sep 17 00:00:00 2001 From: Kyle Drake Date: Wed, 9 Oct 2019 16:22:56 -0700 Subject: [PATCH] direct link to sharded site sitemaps --- Rakefile | 20 +---------- public/robots.txt | 91 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 91 insertions(+), 20 deletions(-) diff --git a/Rakefile b/Rakefile index 5d937b32..825db26a 100644 --- a/Rakefile +++ b/Rakefile @@ -559,29 +559,11 @@ task :generate_sitemap => [:environment] do } } - Zlib::GzipWriter.open File.join(sitemap_root, "index-#{key}.xml.gz") do |gz| + Zlib::GzipWriter.open File.join(sitemap_root, "index-sites-#{key}.xml.gz") do |gz| gz.write builder.to_xml(encoding: 'UTF-8') end } - - # Create root sitemap index that links to the other ones - builder = Nokogiri::XML::Builder.new { |xml| - xml.sitemapindex(xmlns: 'http://www.sitemaps.org/schemas/sitemap/0.9') { - sorted_sites.keys.sort.each { |key| - xml.sitemap { - xml.loc "https://neocities.org/sitemap/index-#{key}.xml.gz" - xml.lastmod Time.now.strftime("%Y-%m-%d") - } - } - } - } - - Zlib::GzipWriter.open File.join(sitemap_root, "index-sites.xml.gz") do |gz| - gz.write builder.to_xml(encoding: 'UTF-8') - end - - # Set basic neocities.org root paths builder = Nokogiri::XML::Builder.new { |xml| xml.urlset(xmlns: 'http://www.sitemaps.org/schemas/sitemap/0.9') { diff --git a/public/robots.txt b/public/robots.txt index 65df08b3..a6117983 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -2,4 +2,93 @@ User-agent: * Allow: / Sitemap: https://neocities.org/sitemap/index-root.xml.gz Sitemap: https://neocities.org/sitemap/index-tags.xml.gz -Sitemap: https://neocities.org/sitemap/index-sites.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-10.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-11.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-12.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-13.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-14.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-15.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-16.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-17.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-18.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-19.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-20.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-21.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-22.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-23.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-24.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-25.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-26.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-27.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-28.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-29.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-30.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-31.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-32.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-33.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-34.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-35.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-36.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-37.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-38.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-39.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-40.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-41.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-42.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-43.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-44.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-45.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-46.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-47.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-48.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-49.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-50.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-51.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-52.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-53.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-54.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-55.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-56.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-57.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-58.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-59.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-60.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-61.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-62.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-63.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-64.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-65.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-66.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-67.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-68.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-69.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-70.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-71.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-72.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-73.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-74.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-75.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-76.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-77.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-78.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-79.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-80.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-81.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-82.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-83.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-84.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-85.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-86.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-87.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-88.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-89.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-90.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-91.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-92.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-93.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-94.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-95.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-96.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-97.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-98.xml.gz +Sitemap: https://neocities.org/sitemap/index-sites-99.xml.gz