Merge pull request #1687 from internetee/log-bounced-emails

Gather data about bounced emails via API

app/controllers/admin/bounced_mail_addresses_controller.rb

@@ -0,0 +1,30 @@
+module Admin
+  class BouncedMailAddressesController < BaseController
+    before_action :set_bounced_mail_address, only: %i[show destroy]
+    load_and_authorize_resource
+
+    # GET /bounced_mail_addresses
+    def index
+      @bounced_mail_addresses = BouncedMailAddress.all.order(created_at: :desc)
+    end
+
+    # GET /bounced_mail_addresses/1
+    def show; end
+
+    # DELETE /bounced_mail_addresses/1
+    def destroy
+      @bounced_mail_address.destroy
+      redirect_to(
+        admin_bounced_mail_addresses_url,
+        notice: 'Bounced mail address was successfully destroyed.'
+      )
+    end
+
+    private
+
+    # Use callbacks to share common setup or constraints between actions.
+    def set_bounced_mail_address
+      @bounced_mail_address = BouncedMailAddress.find(params[:id])
+    end
+  end
+end

app/controllers/api/v1/base_controller.rb

@@ -10,6 +10,11 @@ module Api
         head :unauthorized unless ip_allowed
       end
 
+      def authenticate_shared_key
+        api_key = "Basic #{ENV['api_shared_key']}"
+        head(:unauthorized) unless api_key == request.authorization
+      end
+
       def not_found_error
         uuid = params['uuid']
         json = { error: 'Not Found', uuid: uuid, message: 'Record not found' }

app/controllers/api/v1/bounces_controller.rb

@@ -0,0 +1,25 @@
+module Api
+  module V1
+    class BouncesController < BaseController
+      before_action :authenticate_shared_key
+
+      # POST api/v1/bounces/
+      def create
+        return head(:bad_request) unless bounce_params[:bounce][:bouncedRecipients].any?
+
+        BouncedMailAddress.record(bounce_params)
+        head(:created)
+      rescue ActionController::ParameterMissing
+        head(:bad_request)
+      end
+
+      def bounce_params
+        params.require(:data).require(:bounce).require(:bouncedRecipients).each do |r|
+          r.require(:emailAddress)
+        end
+
+        params.require(:data)
+      end
+    end
+  end
+end

app/controllers/registrar/sessions_controller.rb

@@ -62,6 +62,7 @@ class Registrar
 
     def find_user_by_idc_and_allowed(idc)
       return User.new unless idc
+
       possible_users = ApiUser.where(identity_code: idc) || User.new
       possible_users.each do |selected_user|
         if selected_user.registrar.white_ips.registrar_area.include_ip?(request.ip)

app/models/ability.rb

@@ -109,6 +109,7 @@ class Ability
     can :destroy, :pending
     can :create, :zonefile
     can :access, :settings_menu
+    can :manage, BouncedMailAddress
   end
 
   def static_registrant

app/models/bounced_mail_address.rb

@@ -0,0 +1,28 @@
+class BouncedMailAddress < ApplicationRecord
+  validates :email, :message_id, :bounce_type, :bounce_subtype, :action, :status, presence: true
+
+  def bounce_reason
+    "#{action} (#{status} #{diagnostic})"
+  end
+
+  def self.record(json)
+    bounced_records = json['bounce']['bouncedRecipients']
+    bounced_records.each do |record|
+      bounce_record = BouncedMailAddress.new(params_from_json(json, record))
+
+      bounce_record.save
+    end
+  end
+
+  def self.params_from_json(json, bounced_record)
+    {
+      email: bounced_record['emailAddress'],
+      message_id: json['mail']['messageId'],
+      bounce_type: json['bounce']['bounceType'],
+      bounce_subtype: json['bounce']['bounceSubType'],
+      action: bounced_record['action'],
+      status: bounced_record['status'],
+      diagnostic: bounced_record['diagnosticCode'],
+    }
+  end
+end

app/views/admin/base/_menu.haml

@@ -33,6 +33,7 @@
           %li= link_to t('.blocked_domains'), admin_blocked_domains_path
           %li= link_to t('.reserved_domains'), admin_reserved_domains_path
           %li= link_to t('.disputed_domains'), admin_disputes_path
+          %li= link_to t('.bounced_email_addresses'), admin_bounced_mail_addresses_path
           %li= link_to t('.epp_log'), admin_epp_logs_path(created_after: 'today')
           %li= link_to t('.repp_log'), admin_repp_logs_path(created_after: 'today')
           %li= link_to t('.que'), '/admin/que'

app/views/admin/bounced_mail_addresses/index.html.erb

@@ -0,0 +1,36 @@
+<h1>Bounced Mail Addresses</h1>
+
+<div class="row">
+    <div class="row">
+        <div class="col-md-12">
+            <div class="table-responsive">
+                <table class="table table-hover table-bordered table-condensed">
+                    <thead>
+                        <tr>
+                            <th>Email</th>
+                            <th>Action</th>
+                            <th>Status</th>
+                            <th>Diagnostic</th>
+                            <th>Tracked</th>
+                            <th colspan="2">Actions</th>
+                        </tr>
+                    </thead>
+
+                    <tbody>
+                        <% @bounced_mail_addresses.each do |mail_addr| %>
+                        <tr>
+                            <td><%= mail_addr.email %></td>
+                            <td><%= mail_addr.action %></td>
+                            <td><%= mail_addr.status %></td>
+                            <td><%= mail_addr.diagnostic %></td>
+                            <td><%= mail_addr.created_at %></td>
+                            <td><%= link_to 'Detailed', admin_bounced_mail_address_path(mail_addr) %></td>
+                            <td><%= link_to 'Destroy', admin_bounced_mail_address_path(mail_addr), method: :delete, data: { confirm: 'Are you sure?' } %></td>
+                        </tr>
+                        <% end %>
+                    </tbody>
+                </table>
+            </div>
+        </div>
+    </div>
+</div>

app/views/admin/bounced_mail_addresses/show.html.erb

@@ -0,0 +1,27 @@
+<p>
+  <strong>Email:</strong>
+  <%= @bounced_mail_address.email %>
+</p>
+
+<p>
+  <strong>Bounced message ID:</strong>
+  <%= @bounced_mail_address.message_id %>
+</p>
+
+<p>
+  <strong>Overall bounce type:</strong>
+  <%= @bounced_mail_address.bounce_type %> (<%= @bounced_mail_address.bounce_subtype %> )
+</p>
+
+<p>
+  <strong>Bounced recipient status:</strong>
+  <%= @bounced_mail_address.action %> (<%= @bounced_mail_address.status %>)
+</p>
+
+<p>
+  <strong>Bounced recipient diagnostic:</strong>
+  <pre><%= @bounced_mail_address.diagnostic %></pre>
+</p>
+
+<%= link_to 'Back', admin_bounced_mail_addresses_path %>
+<%= link_to 'Destroy', admin_bounced_mail_address_path(@bounced_mail_address), method: :delete, data: { confirm: 'Are you sure?' } %></td>

config/application.yml.sample

@@ -87,6 +87,9 @@ sk_digi_doc_service_name: 'Testimine'
 registrant_api_base_url:
 registrant_api_auth_allowed_ips: '127.0.0.1, 0.0.0.0' #ips, separated with commas
 
+# Bounces API
+api_shared_key: testkey
+
 # Base URL (inc. https://) of REST registrant portal
 # Leave blank to use internal registrant portal
 registrant_portal_verifications_base_url: ''

config/locales/admin/menu.en.yml

@@ -14,6 +14,7 @@ en:
         blocked_domains: Blocked domains
         reserved_domains: Reserved domains
         disputed_domains: Disputed domains
+        bounced_email_addresses: Bounced emails
         epp_log: EPP log
         repp_log: REPP log
         que: Que

config/routes.rb

@@ -90,7 +90,7 @@ Rails.application.routes.draw do
       end
 
       resources :auctions, only: %i[index show update], param: :uuid
-
+      resources :bounces, only: %i[create]
     end
 
     match '*all', controller: 'cors', action: 'cors_preflight_check', via: [:options],
@@ -320,6 +320,7 @@ Rails.application.routes.draw do
     resources :delayed_jobs
     resources :epp_logs
     resources :repp_logs
+    resources :bounced_mail_addresses, only: %i[index show destroy]
 
     authenticate :admin_user do
       mount Que::Web, at: 'que'

db/migrate/20200916125326_create_bounced_mail_addresses.rb

@@ -0,0 +1,15 @@
+class CreateBouncedMailAddresses < ActiveRecord::Migration[6.0]
+  def change
+    create_table :bounced_mail_addresses do |t|
+      t.string :email, null: false
+      t.string :message_id, null: false
+      t.string :bounce_type, null: false
+      t.string :bounce_subtype, null: false
+      t.string :action, null: false
+      t.string :status, null: false
+      t.string :diagnostic, null: true
+
+      t.timestamps
+    end
+  end
+end

db/structure.sql

@@ -475,6 +475,43 @@ CREATE SEQUENCE public.blocked_domains_id_seq
 ALTER SEQUENCE public.blocked_domains_id_seq OWNED BY public.blocked_domains.id;
 
 
+--
+-- Name: bounced_mail_addresses; Type: TABLE; Schema: public; Owner: -; Tablespace:
+--
+
+CREATE TABLE public.bounced_mail_addresses (
+    id bigint NOT NULL,
+    email character varying NOT NULL,
+    message_id character varying NOT NULL,
+    bounce_type character varying NOT NULL,
+    bounce_subtype character varying NOT NULL,
+    action character varying NOT NULL,
+    status character varying NOT NULL,
+    diagnostic character varying,
+    created_at timestamp(6) without time zone NOT NULL,
+    updated_at timestamp(6) without time zone NOT NULL
+);
+
+
+--
+-- Name: bounced_mail_addresses_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.bounced_mail_addresses_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: bounced_mail_addresses_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.bounced_mail_addresses_id_seq OWNED BY public.bounced_mail_addresses.id;
+
+
 --
 -- Name: certificates; Type: TABLE; Schema: public; Owner: -; Tablespace:
 --
@@ -2658,6 +2695,13 @@ ALTER TABLE ONLY public.bank_transactions ALTER COLUMN id SET DEFAULT nextval('p
 ALTER TABLE ONLY public.blocked_domains ALTER COLUMN id SET DEFAULT nextval('public.blocked_domains_id_seq'::regclass);
 
 
+--
+-- Name: id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.bounced_mail_addresses ALTER COLUMN id SET DEFAULT nextval('public.bounced_mail_addresses_id_seq'::regclass);
+
+
 --
 -- Name: id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -2876,14 +2920,14 @@ ALTER TABLE ONLY public.log_payment_orders ALTER COLUMN id SET DEFAULT nextval('
 
 
 --
--- Name: log_prices id; Type: DEFAULT; Schema: public; Owner: -
+-- Name: id; Type: DEFAULT; Schema: public; Owner: -
 --
 
 ALTER TABLE ONLY public.log_prices ALTER COLUMN id SET DEFAULT nextval('public.log_prices_id_seq'::regclass);
 
 
 --
--- Name: log_registrant_verifications id; Type: DEFAULT; Schema: public; Owner: -
+-- Name: id; Type: DEFAULT; Schema: public; Owner: -
 --
 
 ALTER TABLE ONLY public.log_registrant_verifications ALTER COLUMN id SET DEFAULT nextval('public.log_registrant_verifications_id_seq'::regclass);
@@ -3100,6 +3144,14 @@ ALTER TABLE ONLY public.blocked_domains
     ADD CONSTRAINT blocked_domains_pkey PRIMARY KEY (id);
 
 
+--
+-- Name: bounced_mail_addresses_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace:
+--
+
+ALTER TABLE ONLY public.bounced_mail_addresses
+    ADD CONSTRAINT bounced_mail_addresses_pkey PRIMARY KEY (id);
+
+
 --
 -- Name: certificates_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace:
 --
@@ -3349,7 +3401,7 @@ ALTER TABLE ONLY public.log_payment_orders
 
 
 --
--- Name: log_prices log_prices_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+-- Name: log_prices_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
 
 ALTER TABLE ONLY public.log_prices
@@ -3357,7 +3409,7 @@ ALTER TABLE ONLY public.log_prices
 
 
 --
--- Name: log_registrant_verifications log_registrant_verifications_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+-- Name: log_registrant_verifications_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
 
 ALTER TABLE ONLY public.log_registrant_verifications
@@ -4906,5 +4958,7 @@ INSERT INTO "schema_migrations" (version) VALUES
 ('20200902131603'),
 ('20200908131554'),
 ('20200910085157'),
-('20200910102028');
+('20200910102028'),
+('20200916125326');
+
 

test/fixtures/bounced_mail_addresses.yml

@@ -0,0 +1,10 @@
+one:
+  email: bounced@registry.test
+  message_id: 010f0174a0c7d348-ea6e2fc1-0854-4073-b71f-5cecf9b0d0b2-000000
+  bounce_type: Permanent
+  bounce_subtype: General
+  action: failed
+  status: '5.1.1'
+  diagnostic: 'smtp; 550 5.1.1 user unknown'
+  created_at: <%= Time.zone.parse('2010-07-05').to_s(:db) %>
+  updated_at: <%= Time.zone.parse('2010-07-05').to_s(:db) %>

test/integration/api/v1/bounces/create_test.rb

@@ -0,0 +1,75 @@
+require 'test_helper'
+
+class BouncesApiV1CreateTest < ActionDispatch::IntegrationTest
+  def setup
+    @api_key = "Basic #{ENV['api_shared_key']}"
+    @headers = { "Authorization": "#{@api_key}" }
+    @json_body = { "data": valid_bounce_request }.as_json
+  end
+
+  def test_authorizes_api_request
+    post api_v1_bounces_path, params: @json_body, headers: @headers
+    assert_response :created
+
+    invalid_headers = { "Authorization": "Basic invalid_api_key" }
+    post api_v1_bounces_path, params: @json_body, headers: invalid_headers
+    assert_response :unauthorized
+  end
+
+  def test_returns_bad_request_if_invalid_payload
+    invalid_json_body = @json_body.dup
+    invalid_json_body['data']['bounce']['bouncedRecipients'] = nil
+
+    post api_v1_bounces_path, params: invalid_json_body, headers: @headers
+    assert_response :bad_request
+
+    invalid_json_body = 'aaaa'
+    post api_v1_bounces_path, params: invalid_json_body, headers: @headers
+    assert_response :bad_request
+  end
+
+  def test_saves_new_bounce_object
+    request_body = @json_body.dup
+    random_mail = "#{rand(10000..99999)}@registry.test"
+    request_body['data']['bounce']['bouncedRecipients'][0]['emailAddress'] = random_mail
+
+    post api_v1_bounces_path, params: request_body, headers: @headers
+    assert_response :created
+
+    bounced_mail = BouncedMailAddress.last
+    assert bounced_mail.email = random_mail
+    assert '5.1.1', bounced_mail.status
+    assert 'failed', bounced_mail.action
+  end
+
+  def valid_bounce_request
+    {
+      "notificationType": "Bounce",
+      "mail": {
+        "source": "noreply@registry.test",
+        "sourceIp": "195.43.86.5",
+        "messageId": "010f0174a0c7d348-ea6e2fc1-0854-4073-b71f-5cecf9b0d0b2-000000",
+        "sourceArn": "arn:aws:ses:us-east-2:65026820000:identity/noreply@registry.test",
+        "timestamp": "2020-09-18T10:34:44.000Z",
+        "destination": [ "bounced@registry.test" ],
+        "sendingAccountId": "650268220000"
+      },
+      "bounce": {
+        "timestamp": "2020-09-18T10:34:44.911Z",
+        "bounceType": "Permanent",
+        "feedbackId": "010f0174a0c7d4f9-27d59756-6111-4d5f-xxxx-26bee0d55fa2-000000",
+        "remoteMtaIp": "127.0.01",
+        "reportingMTA": "dsn; xxx.amazonses.com",
+        "bounceSubType": "General",
+        "bouncedRecipients": [
+          {
+            "action": "failed",
+            "status": "5.1.1",
+            "emailAddress": "bounced@registry.test",
+            "diagnosticCode": "smtp; 550 5.1.1 user unknown"
+          }
+        ]
+      }
+    }.as_json
+  end
+end

test/models/bounced_mail_address_test.rb

@@ -0,0 +1,104 @@
+require 'test_helper'
+
+class BouncedMailAddressTest < ActiveSupport::TestCase
+  include ActionMailer::TestHelper
+
+  def setup
+    @bounced_mail = BouncedMailAddress.new
+    @bounced_mail.email = 'recipient@registry.test'
+    @bounced_mail.message_id = '010f0174a0c7d348-ea6e2fc1-0854-4073-b71f-5cecf9b0d0b2-000000'
+    @bounced_mail.bounce_type = 'Permanent'
+    @bounced_mail.bounce_subtype = 'General'
+    @bounced_mail.action = 'failed'
+    @bounced_mail.status = '5.1.1'
+    @bounced_mail.diagnostic =  'smtp; 550 5.1.1 user unknown'
+  end
+
+  def test_email_is_required
+    assert @bounced_mail.valid?
+    @bounced_mail.email = nil
+    assert @bounced_mail.invalid?
+  end
+
+  def test_message_id_is_required
+    assert @bounced_mail.valid?
+    @bounced_mail.message_id = nil
+    assert @bounced_mail.invalid?
+  end
+
+  def test_bounce_type_is_required
+    assert @bounced_mail.valid?
+    @bounced_mail.bounce_type = nil
+    assert @bounced_mail.invalid?
+  end
+
+  def test_bounce_subtype_is_required
+    assert @bounced_mail.valid?
+    @bounced_mail.bounce_subtype = nil
+    assert @bounced_mail.invalid?
+  end
+
+  def test_action_is_required
+    assert @bounced_mail.valid?
+    @bounced_mail.action = nil
+    assert @bounced_mail.invalid?
+  end
+
+  def test_status_is_required
+    assert @bounced_mail.valid?
+    @bounced_mail.status = nil
+    assert @bounced_mail.invalid?
+  end
+
+  def test_diagnostic_is_not_required
+    assert @bounced_mail.valid?
+    @bounced_mail.diagnostic = nil
+    assert @bounced_mail.valid?
+  end
+
+  def test_bounce_reason_is_determined_dynamically
+    assert @bounced_mail.valid?
+    assert_equal 'failed (5.1.1 smtp; 550 5.1.1 user unknown)', @bounced_mail.bounce_reason
+  end
+
+  def test_creates_objects_from_sns_json
+    BouncedMailAddress.record(sns_bounce_payload)
+
+    bounced_mail = BouncedMailAddress.last
+    assert_equal domains(:shop).registrant.email, bounced_mail.email
+    assert_equal 'failed', bounced_mail.action
+    assert_equal '5.1.1', bounced_mail.status
+    assert_equal 'smtp; 550 5.1.1 user unknown', bounced_mail.diagnostic
+  end
+
+  def sns_bounce_payload
+    {
+      "notificationType": "Bounce",
+      "mail": {
+        "source": "noreply@registry.test",
+        "sourceIp": "195.43.86.5",
+        "messageId": "010f0174a0c7d348-ea6e2fc1-0854-4073-b71f-5cecf9b0d0b2-000000",
+        "sourceArn": "arn:aws:ses:us-east-2:65026820000:identity/noreply@registry.test",
+        "timestamp": "2020-09-18T10:34:44.000Z",
+        "destination": [ "#{domains(:shop).registrant.email}" ],
+        "sendingAccountId": "650268220000"
+      },
+      "bounce": {
+        "timestamp": "2020-09-18T10:34:44.911Z",
+        "bounceType": "Permanent",
+        "feedbackId": "010f0174a0c7d4f9-27d59756-6111-4d5f-xxxx-26bee0d55fa2-000000",
+        "remoteMtaIp": "127.0.01",
+        "reportingMTA": "dsn; xxx.amazonses.com",
+        "bounceSubType": "General",
+        "bouncedRecipients": [
+          {
+            "action": "failed",
+            "status": "5.1.1",
+            "emailAddress": "#{domains(:shop).registrant.email}",
+            "diagnosticCode": "smtp; 550 5.1.1 user unknown"
+          }
+        ]
+      }
+    }.as_json
+  end
+end

test/system/admin_area/bounced_mail_addresses_test.rb

@@ -0,0 +1,40 @@
+require 'application_system_test_case'
+
+class AdminBouncedMailAddressesTest < ApplicationSystemTestCase
+  include ActionView::Helpers::NumberHelper
+
+  def setup
+    @bounced_mail = bounced_mail_addresses(:one)
+    @original_default_language = Setting.default_language
+    sign_in users(:admin)
+  end
+
+  def teardown
+    Setting.default_language = @original_default_language
+  end
+
+  def test_shows_bounced_emails
+    visit admin_bounced_mail_addresses_path
+    assert_text @bounced_mail.status
+    assert_text @bounced_mail.action
+    assert_text @bounced_mail.diagnostic
+    assert_text @bounced_mail.email
+  end
+
+  def test_shows_detailed_bounced_email
+    visit admin_bounced_mail_address_path(@bounced_mail)
+    assert_text @bounced_mail.status
+    assert_text @bounced_mail.action
+    assert_text @bounced_mail.diagnostic
+    assert_text @bounced_mail.email
+
+    assert_text @bounced_mail.message_id
+  end
+
+  def test_deletes_registrar
+    visit admin_bounced_mail_address_path(@bounced_mail)
+    click_on 'Destroy'
+
+    assert_text 'Bounced mail address was successfully destroyed.'
+  end
+end