From 8b0d396565e85983cb21ee826985e3f6cb54fedb Mon Sep 17 00:00:00 2001 From: Kyle Drake Date: Sat, 19 Jan 2019 03:02:26 -0800 Subject: [PATCH] Refactor IPFS archiving to support cidv1-base32 --- app/settings.rb | 2 +- app/site.rb | 2 +- ext/base58.rb | 41 --------------------------------- models/archive.rb | 12 ++-------- models/site.rb | 20 ++++++++-------- views/permanent_web.erb | 4 ++-- views/settings/site/profile.erb | 8 +++---- views/site/archives.erb | 5 +--- 8 files changed, 20 insertions(+), 74 deletions(-) delete mode 100644 ext/base58.rb diff --git a/app/settings.rb b/app/settings.rb index 0c53d673..5221c272 100644 --- a/app/settings.rb +++ b/app/settings.rb @@ -53,7 +53,7 @@ post '/settings/:username/profile' do @site.update( profile_comments_enabled: params[:site][:profile_comments_enabled], profile_enabled: params[:site][:profile_enabled], - archiving_disabled: params[:site][:archiving_disabled] + ipfs_archiving_enabled: params[:site][:ipfs_archiving_enabled] ) flash[:success] = 'Profile settings changed.' redirect "/settings/#{@site.username}#profile" diff --git a/app/site.rb b/app/site.rb index 1a0b7bbf..020f99ca 100644 --- a/app/site.rb +++ b/app/site.rb @@ -39,7 +39,7 @@ end get '/site/:username/archives' do @site = Site[username: params[:username]] - not_found if @site.nil? || @site.archiving_disabled + not_found if @site.nil? || !@site.ipfs_archiving_enabled @title = "Site archives for #{@site.title}" @archives = @site.archives_dataset.limit(300).order(:updated_at.desc).all erb :'site/archives' diff --git a/ext/base58.rb b/ext/base58.rb deleted file mode 100644 index 048f7cb8..00000000 --- a/ext/base58.rb +++ /dev/null @@ -1,41 +0,0 @@ -module Base58 - class << self - def int_to_base58(int_val, leading_zero_bytes=0) - alpha = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz" - base58_val, base = '', alpha.size - while int_val > 0 - int_val, remainder = int_val.divmod(base) - base58_val = alpha[remainder] + base58_val - end - base58_val - end - - def base58_to_int(base58_val) - alpha = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz" - int_val, base = 0, alpha.size - base58_val.reverse.each_char.with_index do |char,index| - raise ArgumentError, 'Value not a valid Base58 String.' unless char_index = alpha.index(char) - int_val += char_index*(base**index) - end - int_val - end - - def base58_to_bytestring(base58_val) - [Base58.decode_base58(base58_val)].pack('H*') - end - - def encode_base58(hex) - leading_zero_bytes = (hex.match(/^([0]+)/) ? $1 : '').size / 2 - ("1"*leading_zero_bytes) + int_to_base58( hex.to_i(16) ) - end - - def decode_base58(base58_val) - s = base58_to_int(base58_val).to_s(16); s = (s.bytesize.odd? ? '0'+s : s) - s = '' if s == '00' - leading_zero_bytes = (base58_val.match(/^([1]+)/) ? $1 : '').size - s = ("00"*leading_zero_bytes) + s if leading_zero_bytes > 0 - s - end - alias_method :base58_to_hex, :decode_base58 - end -end diff --git a/models/archive.rb b/models/archive.rb index 23376ea2..bf06406f 100644 --- a/models/archive.rb +++ b/models/archive.rb @@ -5,21 +5,13 @@ class Archive < Sequel::Model set_primary_key [:site_id, :ipfs_hash] unrestrict_primary_key MAXIMUM_ARCHIVES_PER_SITE = 100 - ARCHIVE_WAIT_TIME = 10.minutes + ARCHIVE_WAIT_TIME = 1.minute def before_destroy unpin super end - def self.base58_to_hshca(base58) - Base32.encode(Base58.base58_to_bytestring(base58)).gsub('=', '').downcase - end - - def hshca_hash - self.class.base58_to_hshca ipfs_hash - end - def unpin # Not ideal. An SoA version is in progress. if ENV['RACK_ENV'] == 'production' && $config['ipfs_ssh_host'] && $config['ipfs_ssh_user'] @@ -41,6 +33,6 @@ class Archive < Sequel::Model end def url - "http://#{hshca_hash}.ipfs.neocitiesops.net" + "https://#{ipfs_hash}.ipfs.neocitiesops.net" end end diff --git a/models/site.rb b/models/site.rb index b75648fa..0638b3e3 100644 --- a/models/site.rb +++ b/models/site.rb @@ -707,7 +707,7 @@ class Site < Sequel::Model purge_cache path end - Rye::Cmd.add_command :ipfs, nil, 'add', :r, :Q + Rye::Cmd.add_command :ipfs def add_to_ipfs # Not ideal. An SoA version is in progress. @@ -717,18 +717,21 @@ class Site < Sequel::Model end if $config['ipfs_ssh_host'] && $config['ipfs_ssh_user'] - rbox = Rye::Box.new $config['ipfs_ssh_host'], :user => $config['ipfs_ssh_user'] + rbox = Rye::Box.new $config['ipfs_ssh_host'], user: $config['ipfs_ssh_user'] begin - response = rbox.ipfs "sites/#{self.class.sharding_dir self.username}/#{self.username.gsub(/\/|\.\./, '')}" + cidv0 = rbox.ipfs(:add, :r, :Q, "sites/#{self.class.sharding_dir self.username}/#{self.username.gsub(/\/|\.\./, '')}").first + cidv1b32 = rbox.ipfs(:cid, :base32, cidv0).first ensure rbox.disconnect end else line = Terrapin::CommandLine.new('ipfs', 'add -r -Q :path') - response = line.run path: files_path + response = line.run(path: files_path).strip + line = Terrapin::CommandLine.new('ipfs', 'cid base32 :hash') + cidv1b32 = line.run(hash: response).strip end - response.strip + cidv1b32 end def purge_old_archives @@ -738,11 +741,6 @@ class Site < Sequel::Model end def archive! - #if ENV["RACK_ENV"] == 'test' - # ipfs_hash = "QmcKi2ae3uGb1kBg1yBpsuwoVqfmcByNdMiZ2pukxyLWD8" - #else - #end - ipfs_hash = add_to_ipfs archive = archives_dataset.where(ipfs_hash: ipfs_hash).first @@ -1597,7 +1595,7 @@ class Site < Sequel::Model ] sql.first - unless archiving_disabled + if ipfs_archiving_enabled ArchiveWorker.perform_in Archive::ARCHIVE_WAIT_TIME, self.id end end diff --git a/views/permanent_web.erb b/views/permanent_web.erb index ac2b423f..2282d168 100644 --- a/views/permanent_web.erb +++ b/views/permanent_web.erb @@ -17,7 +17,7 @@

- IPFS archiving is now enabled on all sites. You'll see an IPFS hash link on the site profile, and an archive link that allows you to see past versions of your site (note: this is still a preview, so past site archives may still disappear, but we're working on making it better). + IPFS archiving is now enabled on all sites. You'll see an IPFS CID link on the site profile, and an archive link that allows you to see past versions of your site (note: this is still a preview, so past site archives may still disappear, but we're working on making it better).

@@ -25,7 +25,7 @@

- $ ipfs pin add -r THE_IPFS_HASH_FOR_YOUR_SITE + $ ipfs pin add -r THE_IPFS_CID_FOR_YOUR_SITE

diff --git a/views/settings/site/profile.erb b/views/settings/site/profile.erb index 9789c880..b1a12ab5 100644 --- a/views/settings/site/profile.erb +++ b/views/settings/site/profile.erb @@ -24,10 +24,10 @@

IPFS Archiving

- - checked<% end %> - > Disable IPFS Archiving + + checked<% end %> + > Enable IPFS Archiving
diff --git a/views/site/archives.erb b/views/site/archives.erb index ae0b47b9..a5f708c5 100644 --- a/views/site/archives.erb +++ b/views/site/archives.erb @@ -11,7 +11,7 @@ <% else %> - + <% @archives.each do |archive| %> @@ -28,9 +28,6 @@

Archives are captured once every <%= Archive::ARCHIVE_WAIT_TIME / 60 %> minutes, so if you don't see your latest changes, check back later.

-

- The URLs are using hshca, a standard for using cryptographic hashes with subdomains. -

<% end %>
IPFS Hash (what is this?)IPFS CID (what is this?) Archived Time