Merge branch 'master' into upgrade

app.rb

@@ -77,6 +77,12 @@ before do
   end
 end
 
+after do
+  if @api
+    request.session_options[:skip] = true
+  end
+end
+
 #after do
   #response.headers['Content-Security-Policy'] = %{block-all-mixed-content; default-src 'self'; connect-src 'self' https://api.stripe.com; frame-src https://www.google.com/recaptcha/ https://js.stripe.com; script-src 'self' 'unsafe-inline' https://www.google.com/recaptcha/ https://www.gstatic.com/recaptcha/ https://js.stripe.com; style-src 'self' 'unsafe-inline'; img-src 'self' data: }
 #end

app/api.rb

@@ -5,6 +5,16 @@ get '/api' do
   erb :'api'
 end
 
+post '/api/upload_hash' do
+  require_api_credentials
+  res = {}
+  files = []
+  params.each do |k,v|
+    res[k] = current_site.sha1_hash_match? k, v
+  end
+  api_success files: res
+end
+
 get '/api/list' do
   require_api_credentials
 
@@ -85,7 +95,7 @@ post '/api/delete' do
       api_error 400, 'missing_files', "#{path} was not found on your site, canceled deleting"
     end
 
-    if path == 'index.html'
+    if path == 'index.html' || path == '/index.html'
       api_error 400, 'cannot_delete_index', 'you cannot delete your index.html file, canceled deleting'
     end
 
@@ -110,6 +120,12 @@ get '/api/info' do
   end
 end
 
+get '/api/key' do
+  require_api_credentials
+  current_site.generate_api_key! if current_site.api_key.blank?
+  api_success api_key: current_site.api_key
+end
+
 def api_info_for(site)
   {
     info: {
@@ -148,24 +164,31 @@ def init_api_credentials
   auth = request.env['HTTP_AUTHORIZATION']
 
   begin
-    user, pass = Base64.decode64(auth.match(/Basic (.+)/)[1]).split(':')
+    if bearer_match = auth.match(/^Bearer (.+)/)
+      api_key = bearer_match.captures.first
+      api_error_invalid_auth if api_key.nil? || api_key.empty?
+    else
+      user, pass = Base64.decode64(auth.match(/Basic (.+)/)[1]).split(':')
+    end
   rescue
     api_error_invalid_auth
   end
 
-  if Site.valid_login? user, pass
+  if defined?(api_key) && !api_key.blank?
-    site = Site[username: user]
+    site = Site[api_key: api_key]
-
+  elsif defined?(user) && defined?(pass)
-    if site.nil? || site.is_banned
+    site = Site.get_site_from_login user, pass
-      api_error_invalid_auth
-    end
-
-    DB['update sites set api_calls=api_calls+1 where id=?', site.id].first
-
-    session[:id] = site.id
   else
     api_error_invalid_auth
   end
+
+  if site.nil? || site.is_banned || site.is_deleted
+    api_error_invalid_auth
+  end
+
+  DB['update sites set api_calls=api_calls+1 where id=?', site.id].first
+
+  session[:id] = site.id
 end
 
 def api_success(message_or_obj)
@@ -189,7 +212,7 @@ def api_error(status, error_type, message)
 end
 
 def api_error_invalid_auth
-  api_error 403, 'invalid_auth', 'invalid credentials - please check your username and password'
+  api_error 403, 'invalid_auth', 'invalid credentials - please check your username and password (or your api key)'
 end
 
 def api_not_found

app/password_reset.rb

@@ -21,7 +21,7 @@ post '/send_password_reset' do
       body = <<-EOT
 Hello! This is the Neocities cat, and I have received a password reset request for your e-mail address.
 
-Go to this URL to reset your password: http://neocities.org/password_reset_confirm?username=#{Rack::Utils.escape(site.username)}&token=#{token}
+Go to this URL to reset your password: https://neocities.org/password_reset_confirm?username=#{Rack::Utils.escape(site.username)}&token=#{token}
 
 If you didn't request this password reset, you can ignore it. Or hide under a bed. Or take a nap. Your call.
 

app/settings.rb

@@ -168,6 +168,17 @@ post '/settings/:username/custom_domain' do
   end
 end
 
+post '/settings/:username/generate_api_key' do
+  require_login
+  require_ownership_for_settings
+  is_new = current_site.api_key.nil?
+  current_site.generate_api_key!
+
+  msg = is_new ? "New API key has been generated." : "API key has been regenerated."
+  flash[:success] = msg
+  redirect "/settings/#{current_site.username}#api_key"
+end
+
 post '/settings/change_password' do
   require_login
 

config.yml.template

@@ -50,3 +50,4 @@ test:
     - mrteacher
   stop_forum_spam_api_key: testkey
   screenshots_url: http://screenshots:derp@screenshotssite.com
+  cache_control_ip: 1.2.3.4

config.yml.travis

@@ -20,3 +20,4 @@ education_tag_whitelist:
   - mrteacher
 stop_forum_spam_api_key: testkey
 screenshots_url: http://screenshots:derp@screenshotssite.com
+cache_control_ip: 1.2.3.4

migrations/103_sites_api_key.rb

@@ -0,0 +1,9 @@
+Sequel.migration do
+  up {
+    DB.add_column :sites, :api_key, :text
+  }
+
+  down {
+    DB.drop_column :sites, :api_key
+  }
+end

models/site.rb

@@ -70,6 +70,7 @@ class Site < Sequel::Model
   THUMBNAIL_RESOLUTIONS  = ['210x158']
 
   MAX_FILE_SIZE = 10**8 # 100 MB
+  MAX_SITE_DOWNLOAD_SIZE = 2_000_000_000 # 2GB
 
   CLAMAV_THREAT_MATCHES = [
     /^VBS/,
@@ -288,11 +289,8 @@ class Site < Sequel::Model
 
     def get_site_from_login(username_or_email, plaintext)
       site = get_with_identifier username_or_email
-
+      return nil if site.nil? || site.is_deleted || site.is_banned || !site.valid_password?(plaintext)
-      return false if site.nil?
+      site
-      return false if site.is_deleted
-      return false if site.is_banned
-      site.valid_password?(plaintext) ? site : nil
     end
 
     def bcrypt_cost
@@ -1138,6 +1136,10 @@ class Site < Sequel::Model
     ((total_space_used.to_f / maximum_space) * 100).round(1)
   end
 
+  def too_big_to_download?
+    space_used > MAX_SITE_DOWNLOAD_SIZE
+  end
+
   # Note: Change Stat#prune! and the nginx map compiler if you change this business logic.
   def supporter?
     owner.plan_type != 'free'
@@ -1499,6 +1501,17 @@ class Site < Sequel::Model
     true
   end
 
+  def generate_api_key!
+    self.api_key = SecureRandom.hex(16)
+    save_changes validate: false
+  end
+
+  def sha1_hash_match?(path, sha1_hash)
+    relative_path = scrubbed_path path
+    site_file = site_files_dataset.where(path: relative_path, sha1_hash: sha1_hash).first
+    !site_file.nil?
+  end
+
   private
 
   def store_file(path, uploaded, opts={})

models/stat.rb

@@ -19,7 +19,7 @@ class Stat < Sequel::Model
     def parse_logfiles(path)
       total_site_stats = {}
 
-      cache_control_ip = Resolv::DNS.new.getaddress('neocities.org')
+      cache_control_ip = $config['cache_control_ip']
 
       Dir["#{path}/*.log"].each do |log_path|
         site_logs = {}
@@ -276,4 +276,3 @@ end
     end
   end
 =end
-

tests/api_tests.rb

@@ -197,6 +197,50 @@ describe 'api delete' do
   end
 end
 
+describe 'api key' do
+  it 'generates new key with valid login' do
+    create_site
+    basic_authorize @user, @pass
+    get '/api/key'
+    res[:result].must_equal 'success'
+    res[:api_key].must_equal @site.reload.api_key
+  end
+
+  it 'returns existing key' do
+    create_site
+    @site.generate_api_key!
+    basic_authorize @user, @pass
+    get '/api/key'
+    res[:api_key].must_equal @site.api_key
+  end
+
+  it 'fails for bad login' do
+    create_site
+    basic_authorize 'zero', 'cool'
+    get '/api/key'
+    res[:error_type].must_equal 'invalid_auth'
+  end
+end
+
+describe 'api upload hash' do
+  it 'succeeds' do
+    create_site
+    basic_authorize @user, @pass
+    test_hash = Digest::SHA1.file('./tests/files/test.jpg').hexdigest
+
+    post '/api/upload', {
+      'test.jpg' => Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg'),
+      'test2.jpg' => Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg')
+    }
+
+    post '/api/upload_hash', "test.jpg" => test_hash, "test2.jpg" => Digest::SHA1.hexdigest('herpderp')
+
+    res[:result].must_equal 'success'
+    res[:files][:'test.jpg'].must_equal true
+    res[:files][:'test2.jpg'].must_equal false
+  end
+end
+
 describe 'api upload' do
   it 'fails with no auth' do
     post '/api/upload'
@@ -217,6 +261,26 @@ describe 'api upload' do
     res[:error_type].must_equal 'missing_files'
   end
 
+  it 'succeeds with valid api key' do
+    create_site
+    @site.api_key.must_equal nil
+    @site.generate_api_key!
+    @site.reload.api_key.wont_equal nil
+    header 'Authorization', "Bearer #{@site.api_key}"
+    post '/api/upload', 'test.jpg' => Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg')
+    res[:result].must_equal 'success'
+    site_file_exists?('test.jpg').must_equal true
+  end
+
+  it 'fails with bad api key' do
+    create_site
+    @site.generate_api_key!
+    header 'Authorization', "Bearer zerocool"
+    post '/api/upload', 'test.jpg' => Rack::Test::UploadedFile.new('./tests/files/test.jpg', 'image/jpeg')
+    res[:result].must_equal 'error'
+    res[:error_type].must_equal 'invalid_auth'
+  end
+
 =begin
   # Getting too slow to run this test
   it 'fails with too many files' do

views/_footer.erb

@@ -3,7 +3,7 @@
 
     <div class="row">
       <p class="tiny col credits">
-        <a href="http://neocities.org" title="Neocities.org" style="text-decoration:none;">Neocities</a> is <a href="https://github.com/neocities" title="Neocities on GitHub">open source</a>. Follow us on <a href="https://twitter.com/neocities">Twitter</a> or <a href="https://www.facebook.com/neocities">Facebook</a>
+        <a href="https://neocities.org" title="Neocities.org" style="text-decoration:none;">Neocities</a> is <a href="https://github.com/neocities" title="Neocities on GitHub">open source</a>. Follow us on <a href="https://twitter.com/neocities">Twitter</a> or <a href="https://www.facebook.com/neocities">Facebook</a>
       </p>
 
       <nav class="footer-Nav col">

views/_news.erb

@@ -109,7 +109,7 @@
             <% site_change_filenames.each do |f| %>
               <div class="file">
                 <div class="html-thumbnail <%= site_change_file_display_class f %>">
-                  <a href="http://<%= event_site.host %><%= f == 'index.html' ? '' : "/#{f}" %>">
+                  <a href="https://<%= event_site.host %><%= f == 'index.html' ? '' : "/#{f}" %>">
                     <% if site_change_file_display_class(f) == 'html' %>
                       <img src="<%= event_site.screenshot_url(f, '540x405') %>">
                     <% elsif site_change_file_display_class(f) == 'image' %>

views/_share.erb

@@ -5,16 +5,16 @@
 %>
 <a href="/site/<%= site.username %>.rss" target="_blank"><span>RSS/Atom Feed</span></a>
 <br>
-<a href="http://facebook.com/sharer.php?u=<%= Rack::Utils.build_query(u: "#{page_uri}") %>" target="_blank">Facebook</a>
+<a href="https://facebook.com/sharer.php?u=<%= Rack::Utils.build_query(u: "#{page_uri}") %>" target="_blank">Facebook</a>
 <br>
 <a href="https://twitter.com/intent/tweet?<%= Rack::Utils.build_query(text: "#{site.title}: #{page_uri}") %>" target="_blank">Twitter</a>
 <br>
-<a href="http://www.reddit.com/submit?<%= Rack::Utils.build_query(title: "#{site.title}", url: "#{page_uri}" )%>" target="_blank">Reddit</a>
+<a href="https://www.reddit.com/submit?<%= Rack::Utils.build_query(title: "#{site.title}", url: "#{page_uri}" )%>" target="_blank">Reddit</a>
 <br>
-<a href="http://www.tumblr.com/share?<%= Rack::Utils.build_query(v: 3, u: "#{page_uri}", t: "#{site.title}") %>" target="_blank">Tumblr</a>
+<a href="https://www.tumblr.com/share?<%= Rack::Utils.build_query(v: 3, u: "#{page_uri}", t: "#{site.title}") %>" target="_blank">Tumblr</a>
 <br>
-<a href="http://www.stumbleupon.com/submit?<%= Rack::Utils.build_query(url: "#{page_uri}", title: "#{site.title}") %>" target="_blank">StumbleUpon</a>
+<a href="https://www.stumbleupon.com/submit?<%= Rack::Utils.build_query(url: "#{page_uri}", title: "#{site.title}") %>" target="_blank">StumbleUpon</a>
 <br>
-<a href="http://del.icio.us/post?<%= Rack::Utils.build_query(url: "#{page_uri}", title: "#{site.title}") %>" target="_blank">Del.ici.ous</a>
+<a href="https://del.icio.us/post?<%= Rack::Utils.build_query(url: "#{page_uri}", title: "#{site.title}") %>" target="_blank">Del.ici.ous</a>
 <br>
 <a href="https://plus.google.com/share?<%= Rack::Utils.build_query(url: "#{page_uri}") %>" target="_blank">Google+</a>

views/api.erb

@@ -177,6 +177,29 @@ var api = new neocities('YOURUSERNAME', 'YOURPASSWORD')
 api.info('madamfrp', function(resp) {
   console.log(resp)
 })</code></pre>
+
+
+
+<h2>GET /api/key</h2>
+<p>
+  Returns an API key that you can use for the API instead of login credentials.
+  It will automatically generate a new API key if one doesn't exist yet for your site.
+</p>
+<h3>Examples</h3>
+<h6>Using cURL</h6>
+<pre><code class="bash">$ curl "https://USER:PASS@neocities.org/api/key"
+{
+  "result": "success",
+  "api_key": "da77c3530c30593663bf7b797323e48c"
+}</code></pre>
+
+  <p>Using the api key for requests:</p>
+
+  <pre><code class="bash">$ curl -H "Authorization: Bearer da77c3530c30593663bf7b797323e48c" \
+https://neocities.org/api/info</code></pre>
+
   <h2>Need something the API doesn't provide?</h2>
-  <p>If the API does not supply something you need, <a href="/contact">contact us</a> and we will try to add it!
+  <p>
+    If the API does not supply something you need, <a href="/contact">contact us</a> and we will try to add it!
+  </p>
 </div>

views/dashboard.erb

@@ -158,7 +158,7 @@
               <% if file[:is_directory] %>
               <a class="link-overlay" href="?dir=<%= Rack::Utils.escape file[:path] %>" title="View <%= file[:path] %>"></a>
               <% else %>
-              <a class="link-overlay" href="http://<%= current_site.username %>.neocities.org<%= file[:path] == '/index.html' ? '/' : file[:path] %>" title="View <%= file[:path] == '/index.html' ? 'your site' : file[:path] %>" target="_blank"></a>
+              <a class="link-overlay" href="https://<%= current_site.username %>.neocities.org<%= file[:path] == '/index.html' ? '/' : file[:path] %>" title="View <%= file[:path] == '/index.html' ? 'your site' : file[:path] %>" target="_blank"></a>
               <% end %>
             </div>
           </div>
@@ -190,9 +190,11 @@
   <% if !current_site.plan_feature(:no_file_restrictions) %>
     <a href="/site_files/allowed_types">Allowed file types</a> |
   <% end %>
-  <a href="/site_files/<%= current_site.username %>.zip">Download entire site</a>
+    <% unless current_site.too_big_to_download? %>
+      <a href="/site_files/<%= current_site.username %>.zip">Download entire site</a> |
+    <% end %>
   <% unless is_education? %>
-   | <a href="/site_files/mount_info">Mount your site as a drive on your computer!</a>
+    <a href="/site_files/mount_info">Mount your site as a drive on your computer</a>
   <% end %>
 </div>
 

views/permanent_web.erb

@@ -9,7 +9,7 @@
   <img src="/img/neocities-ipfs.jpg" style="margin-bottom: 20px">
   <article role="article">
     <p>
-      Neocities has launched an experimental implementation of <a href="http://ipfs.io">IPFS</a>. IPFS is short for the "Inter-Planetary File System", and is the foundation for a new way to distribute web content that is being called The Permanent Web. The idea behind the Permanent Web is simple: Instead of serving web sites from central servers, we believe that web serving should be decentralized, and that IPFS is an eventual replacement to the aging HTTP protocol for serving static web sites.
+      Neocities has launched an experimental implementation of <a href="https://ipfs.io">IPFS</a>. IPFS is short for the "Inter-Planetary File System", and is the foundation for a new way to distribute web content that is being called The Permanent Web. The idea behind the Permanent Web is simple: Instead of serving web sites from central servers, we believe that web serving should be decentralized, and that IPFS is an eventual replacement to the aging HTTP protocol for serving static web sites.
     </p>
 
     <p>
@@ -21,7 +21,7 @@
     </p>
 
     <p>
-      If you want to play around with this new technology, you can get IPFS for your computer and use it to retrieve content from our IPFS node servers. All you need to do is <a href="http://ipfs.io/docs/install/">download the IPFS daemon</a> (OSX/Linux only for now), and run the following command in your terminal:
+      If you want to play around with this new technology, you can get IPFS for your computer and use it to retrieve content from our IPFS node servers. All you need to do is <a href="https://ipfs.io/docs/install/">download the IPFS daemon</a> (OSX/Linux only for now), and run the following command in your terminal:
     </p>
 
     <p>

views/settings/site.erb

@@ -29,6 +29,8 @@
           <li><a href="#username" data-toggle="tab">Change Site (User) Name</a></li>
           <li><a href="#tipping" data-toggle="tab">Tipping</a></li>
 
+          <li><a href="#api_key" data-toggle="tab">API Key</a></li>
+
           <% if @site.admin_nsfw != true %>
             <li><a href="#nsfw" data-toggle="tab">18+</a></li>
           <% end %>
@@ -53,7 +55,9 @@
           <div class="tab-pane" id="tipping">
             <%== erb :'settings/site/tipping' %>
           </div>
-
+          <div class="tab-pane" id="api_key">
+            <%== erb :'settings/site/api_key' %>
+          </div>
           <% if @site.admin_nsfw != true %>
             <div class="tab-pane" id="nsfw">
               <%== erb :'settings/site/nsfw' %>

views/settings/site/_api_key_form.erb

@@ -0,0 +1,4 @@
+<form method="POST" action="/settings/<%= @site.username %>/generate_api_key">
+  <%== csrf_token_input_html %>
+  <input class="btn-Action" type="submit" value="Generate API Key">
+</form>

views/settings/site/api_key.erb

@@ -0,0 +1,19 @@
+<h2>API Key</h2>
+
+<p>
+  An API Key can be used with the <a href="https://neocities.org/api">Neocities API</a> to allow for changes to your site without requiring login credentials. <strong>This API key allows full write access to your site. Keep it secret, keep it safe.</strong>
+</p>
+
+<% if @site.api_key %>
+  <p>Your API key:<br><strong><%= @site.api_key %></strong></p>
+
+  <p>
+    If you need to regenerate the key for some reason, you can do that below.
+    Keep in mind that this will make the old key no longer function.
+  </p>
+
+  <%== erb :'settings/site/_api_key_form', layout: false %>
+<% else %>
+  <p>You haven't yet generated an API key, click the button to create one:</p>
+  <%== erb :'settings/site/_api_key_form', layout: false %>
+<% end %>

views/site_files/mount_info.erb

@@ -1,7 +1,7 @@
 <div class="header-Outro">
   <div class="row content single-Col">
   <h1>Mount your site</h1>
-  <h3 class="subtitle">Now you can access your Neocities site as a drive on your computer!</h3>
+  <h3 class="subtitle">Access your Neocities site as a drive on your computer</h3>
   </div>
 </div>
 
@@ -10,7 +10,7 @@
     <h1>About</h1>
 
     <p>
-      Neocities now supports <strong>WebDAV</strong>, which allows you to mount your Neocities share on your computer. Now you can access and change your files on your own computer's file manager!
+      Neocities supports <strong>WebDAV</strong>, which allows you to mount your Neocities share on your computer.
     </p>
 
     <% if current_site.nil? %>
@@ -22,20 +22,24 @@
         This feature requires a supporter account. <a href="/supporter">Click here</a> to become a supporter.
       </p>
     <% else %>
-      <h2>Instructions for Windows 7</h2>
+      <h2>Instructions for Windows</h2>
 
       <p>
-        <a href="http://www.onemetric.com.au/Documentation/Mounting-A-WebDAV-Share-Windows-7">Use these instructions</a>, except use this URL to connect: <strong>https://neocities.org/webdav</strong>
+        Unfortunately, the WebDAV that comes with Windows file manager has issues with SSL,
-        <br>
+        and the problem has not yet been fixed. We recommend using a client like <a href="https://cyberduck.io">Cyberduck</a> with Windows, which is free and has
-        Enter your login info when requested.
+        WebDAV support. You can also try <a href="https://mountainduck.io/">Mountain Duck</a>
-      </p>
+        which will allow you to mount your Neocities site as a hard drive on your computer.
 
       <h2>Instructions for OSX</h2>
 
       <p>
-      <a href="http://support.apple.com/kb/ph3857">Use these instructions</a>, and use this URL to connect: <strong>https://neocities.org/webdav</strong>
+        <a href="http://support.apple.com/kb/ph3857">Use these instructions</a>, and use this URL to connect: <strong>https://neocities.org/webdav</strong>
-      <br>
+        <br>
-      Enter login info when requested.
+        Enter login info when requested.
+      </p>
+
+      <p>
+        You can also try out <a href="https://cyberduck.io">Cyberduck</a> or <a href="https://mountainduck.io/">Mountain Duck</a> if you run into any issues.
       </p>
 
       <h2>Instructions for Linux (Ubuntu)</h2>

views/templates/index.erb

@@ -11,7 +11,7 @@
   <body>
     <h1>Welcome to my Website!</h1>
 
-    <p>This is a paragraph! Here's how you make a link: <a href="http://neocities.org">Neocities</a>.</p>
+    <p>This is a paragraph! Here's how you make a link: <a href="https://neocities.org">Neocities</a>.</p>
 
     <p>Here's how you can make <strong>bold</strong> and <em>italic</em> text.</p>
 
@@ -26,6 +26,6 @@
       <li>Third thing</li>
     </ul>
 
-    <p>To learn more HTML/CSS, check out these <a href="http://neocities.org/tutorials">tutorials</a>!</p>
+    <p>To learn more HTML/CSS, check out these <a href="https://neocities.org/tutorials">tutorials</a>!</p>
   </body>
 </html>