Clean up issues with RDAP redaction (#2067)

Instead of using REDACTED FOR PRIVACY everywhere we should just include
the empty string (this is what the spec says, what other gTLD registrars
do, and what the RDAP conformance tool at
https://github.com/icann/rdap-conformance-tool says to do.

In the contact VCards, we omit redacted fields entirely unless the spec
requires that they exist (the version number and an empty 'fn' field).
This also applies to the "handle" field.

Eventually we will probably want to add the redaction extension but
that's not RFCed yet and isn't required for the August RDAP conformance
deadline.

core/src/main/java/google/registry/rdap/RdapIcannStandardInformation.java

@@ -149,14 +149,6 @@ public class RdapIcannStandardInformation {
                   .build())
           .build();
 
-  /**
-   * String that replaces GDPR redacted values.
-   *
-   * <p>GTLD Registration Data Temp Spec 17may18, Appendix A, 2.2: Fields required to be "redacted"
-   * MUST privide in the value section text similar to "REDACTED FOR PRIVACY"
-   */
-  static final String CONTACT_REDACTED_VALUE = "REDACTED FOR PRIVACY";
-
   /**
    * Included in ALL contact responses, even if the user is authorized.
    *

core/src/main/java/google/registry/rdap/RdapJsonFormatter.java

@@ -21,7 +21,6 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static com.google.common.collect.ImmutableSetMultimap.toImmutableSetMultimap;
 import static google.registry.model.EppResourceUtils.isLinked;
 import static google.registry.persistence.transaction.TransactionManagerFactory.replicaTm;
-import static google.registry.rdap.RdapIcannStandardInformation.CONTACT_REDACTED_VALUE;
 import static google.registry.util.CollectionUtils.union;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -39,7 +38,6 @@ import com.google.gson.JsonArray;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.model.EppResource;
 import google.registry.model.contact.Contact;
-import google.registry.model.contact.ContactAddress;
 import google.registry.model.contact.ContactPhoneNumber;
 import google.registry.model.contact.PostalInfo;
 import google.registry.model.domain.DesignatedContact;
@@ -519,69 +517,68 @@ public class RdapJsonFormatter {
     boolean isAuthorized =
         rdapAuthorization.isAuthorizedForRegistrar(contact.getCurrentSponsorRegistrarId());
 
-    // ROID needs to be redacted if we aren't authorized, so we can't have a self-link for
+    VcardArray.Builder vcardBuilder = VcardArray.builder();
-    // unauthorized users
+
     if (isAuthorized) {
-      contactBuilder.linksBuilder().add(makeSelfLink("entity", contact.getRepoId()));
+      fillRdapContactEntityWhenAuthorized(contactBuilder, vcardBuilder, contact, outputDataType);
-    }
+    } else {
-
+      // GTLD Registration Data Temp Spec 17may18, Appendix A, 2.3, 2.4 and RDAP Response Profile
-    // Only show the "summary data remark" if the user is authorized to see this data - because
+      // 2.7.4.1, 2.7.4.2 - the following fields must be redacted:
-    // unauthorized users don't have a self link meaning they can't navigate to the full data.
+      // for REGISTRANT:
-    if (outputDataType != OutputDataType.FULL && isAuthorized) {
+      // handle (ROID), FN (name), TEL (telephone/fax and extension), street, city, postal code
-      contactBuilder.remarksBuilder().add(RdapIcannStandardInformation.SUMMARY_DATA_REMARK);
+      // for ADMIN, TECH:
-    }
+      // handle (ROID), FN (name), TEL (telephone/fax and extension), Organization, street, city,
-
+      // state/province, postal code, country
-    // GTLD Registration Data Temp Spec 17may18, Appendix A, 2.3, 2.4 and RDAP Response Profile
+      //
-    // 2.7.4.1, 2.7.4.2 - the following fields must be redacted:
+      // Note that in theory we have to show the Organization and state/province and country for the
-    // for REGISTRANT:
+      // REGISTRANT. For now, we won't do that until we make sure it's really OK for GDPR
-    // handle (ROID), FN (name), TEL (telephone/fax and extension), street, city, postal code
+      //
-    // for ADMIN, TECH:
-    // handle (ROID), FN (name), TEL (telephone/fax and extension), Organization, street, city,
-    // state/province, postal code, country
-    //
-    // Note that in theory we have to show the Organization and state/province and country for the
-    // REGISTRANT. For now, we won't do that until we make sure it's really OK for GDPR
-    //
-    if (!isAuthorized) {
       // RDAP Response Profile 2.7.4.3: if we redact values from the contact, we MUST include a
       // remark
       contactBuilder
           .remarksBuilder()
           .add(RdapIcannStandardInformation.CONTACT_PERSONAL_DATA_HIDDEN_DATA_REMARK);
-      // to make sure we don't accidentally display data we shouldn't - we replace the
+      contactBuilder.setHandle("");
-      // contact with a safe resource. Then we can add any information we need (e.g. the
+      // The VCard format requires a "fn" entry even if it is empty (redacted)
-      // Organization / state / country of the registrant), although we currently don't do that.
+      vcardBuilder.add(Vcard.create("fn", "text", ""));
-      contact =
-          new Contact.Builder()
-              .setRepoId(CONTACT_REDACTED_VALUE)
-              .setVoiceNumber(
-                  new ContactPhoneNumber.Builder().setPhoneNumber(CONTACT_REDACTED_VALUE).build())
-              .setFaxNumber(
-                  new ContactPhoneNumber.Builder().setPhoneNumber(CONTACT_REDACTED_VALUE).build())
-              .setInternationalizedPostalInfo(
-                  new PostalInfo.Builder()
-                      .setName(CONTACT_REDACTED_VALUE)
-                      .setOrg(CONTACT_REDACTED_VALUE)
-                      .setType(PostalInfo.Type.INTERNATIONALIZED)
-                      .setAddress(
-                          new ContactAddress.Builder()
-                              .setStreet(ImmutableList.of(CONTACT_REDACTED_VALUE))
-                              .setCity(CONTACT_REDACTED_VALUE)
-                              .setState(CONTACT_REDACTED_VALUE)
-                              .setZip(CONTACT_REDACTED_VALUE)
-                              .setCountryCode("XX")
-                              .build())
-                      .build())
-              .build();
     }
 
-    // RDAP Response Profile 2.7.3 - we MUST provide a handle set with the ROID, subject to the
+    contactBuilder.setVcardArray(vcardBuilder.build());
-    // redaction above.
+    contactBuilder.rolesBuilder().addAll(roles);
-    contactBuilder.setHandle(contact.getRepoId());
 
-    // RDAP Response Profile doesn't mention status for contacts, so we only show it if we're both
+    // RDAP Response Profile 2.7.5.1, 2.7.5.3:
-    // FULL and Authorized.
+    // email MUST be omitted, and we MUST have a Remark saying so
-    if (outputDataType == OutputDataType.FULL && isAuthorized) {
+    contactBuilder
+        .remarksBuilder()
+        .add(RdapIcannStandardInformation.CONTACT_EMAIL_REDACTED_FOR_DOMAIN);
+
+    if (outputDataType != OutputDataType.INTERNAL) {
+      // Rdap Response Profile 2.7.6 must have "last update of RDAP database" response. But this is
+      // only for direct query responses and not for internal objects. I'm not sure why it's in that
+      // section at all...
+      contactBuilder.setLastUpdateOfRdapDatabaseEvent(
+          Event.builder()
+              .setEventAction(EventAction.LAST_UPDATE_OF_RDAP_DATABASE)
+              .setEventDate(getRequestTime())
+              .build());
+    }
+    return contactBuilder.build();
+  }
+
+  private void fillRdapContactEntityWhenAuthorized(
+      RdapContactEntity.Builder contactBuilder,
+      VcardArray.Builder vcardBuilder,
+      Contact contact,
+      OutputDataType outputDataType) {
+    // ROID needs to be redacted if we aren't authorized, so we can't have a self-link for
+    // unauthorized users
+    contactBuilder.linksBuilder().add(makeSelfLink("entity", contact.getRepoId()));
+    // RDAP Response Profile 2.7.3 - we MUST provide a handle set with the ROID, subject to
+    // redaction.
+    contactBuilder.setHandle(contact.getRepoId());
+    if (outputDataType.equals(OutputDataType.FULL)) {
+      // RDAP Response Profile doesn't mention status for contacts, so we only show it if we're both
+      // FULL and Authorized.
       contactBuilder
           .statusBuilder()
           .addAll(
@@ -591,12 +588,18 @@ public class RdapJsonFormatter {
                       : contact.getStatusValues(),
                   false,
                   contact.getDeletionTime().isBefore(getRequestTime())));
+      // If we are outputting all data (not just summary data), also add events taken from the
+      // history entries. This isn't strictly required.
+      //
+      // We also only add it for authorized users because millisecond times can fingerprint a user
+      // just as much as the handle can.
+      contactBuilder.eventsBuilder().addAll(makeOptionalEvents(contact));
+    } else {
+      // Only show the "summary data remark" if the user is authorized to see this data - because
+      // unauthorized users don't have a self link meaning they can't navigate to the full data.
+      contactBuilder.remarksBuilder().add(RdapIcannStandardInformation.SUMMARY_DATA_REMARK);
     }
-
+    // Adding the VCard members when not redacted.
-    contactBuilder.rolesBuilder().addAll(roles);
-
-    VcardArray.Builder vcardBuilder = VcardArray.builder();
-    // Adding the VCard members subject to the redaction above.
     //
     // RDAP Response Profile 2.7.3 - we MUST have FN, ADR, TEL, EMAIL.
     //
@@ -622,33 +625,6 @@ public class RdapJsonFormatter {
     if (faxPhoneNumber != null) {
       vcardBuilder.add(makePhoneEntry(PHONE_TYPE_FAX, makePhoneString(faxPhoneNumber)));
     }
-    // RDAP Response Profile 2.7.5.1, 2.7.5.3:
-    // email MUST be omitted, and we MUST have a Remark saying so
-    contactBuilder
-        .remarksBuilder()
-        .add(RdapIcannStandardInformation.CONTACT_EMAIL_REDACTED_FOR_DOMAIN);
-    contactBuilder.setVcardArray(vcardBuilder.build());
-
-    if (outputDataType != OutputDataType.INTERNAL) {
-      // Rdap Response Profile 2.7.6 must have "last update of RDAP database" response. But this is
-      // only for direct query responses and not for internal objects. I'm not sure why it's in that
-      // section at all...
-      contactBuilder.setLastUpdateOfRdapDatabaseEvent(
-          Event.builder()
-              .setEventAction(EventAction.LAST_UPDATE_OF_RDAP_DATABASE)
-              .setEventDate(getRequestTime())
-              .build());
-    }
-
-    // If we are outputting all data (not just summary data), also add events taken from the history
-    // entries. This isn't strictly required.
-    //
-    // We also only add it for authorized users because millisecond times can fingerprint a user
-    // just as much as the handle can.
-    if (outputDataType == OutputDataType.FULL && isAuthorized) {
-      contactBuilder.eventsBuilder().addAll(makeOptionalEvents(contact));
-    }
-    return contactBuilder.build();
   }
 
   /**

core/src/test/resources/google/registry/rdap/rdap_associated_contact_no_personal_data.json

@@ -5,7 +5,7 @@
     "icann_rdap_technical_implementation_guide_0"
   ],
   "objectClassName" : "entity",
-  "handle" : "REDACTED FOR PRIVACY",
+  "handle" : "",
   "events": [
     {
       "eventAction": "last update of RDAP database",
@@ -15,25 +15,8 @@
   "vcardArray": [
     "vcard",
     [
-      ["version",{},"text","4.0"],
+      ["version", {}, "text", "4.0"],
-      ["fn",{},"text","REDACTED FOR PRIVACY"],
+      ["fn", {}, "text", ""]
-      ["org",{},"text","REDACTED FOR PRIVACY"],
-      [
-        "adr",
-        {},
-        "text",
-        [
-          "",
-          "",
-          "REDACTED FOR PRIVACY",
-          "REDACTED FOR PRIVACY",
-          "REDACTED FOR PRIVACY",
-          "REDACTED FOR PRIVACY",
-          "XX"
-        ]
-      ],
-      ["tel",{"type":["voice"]},"uri","tel:REDACTED FOR PRIVACY"],
-      ["tel",{"type":["fax"]},"uri","tel:REDACTED FOR PRIVACY"]
     ]
   ],
   "remarks": [

core/src/test/resources/google/registry/rdap/rdap_contact_no_personal_data_with_remark.json

@@ -5,7 +5,7 @@
     "icann_rdap_technical_implementation_guide_0"
   ],
   "objectClassName" : "entity",
-  "handle" : "REDACTED FOR PRIVACY",
+  "handle" : "",
   "events": [
     {
       "eventAction": "last update of RDAP database",
@@ -15,25 +15,8 @@
   "vcardArray": [
     "vcard",
     [
-      ["version",{},"text","4.0"],
+      ["version", {}, "text", "4.0"],
-      ["fn",{},"text","REDACTED FOR PRIVACY"],
+      ["fn", {}, "text", ""]
-      ["org",{},"text","REDACTED FOR PRIVACY"],
-      [
-        "adr",
-        {},
-        "text",
-        [
-          "",
-          "",
-          "REDACTED FOR PRIVACY",
-          "REDACTED FOR PRIVACY",
-          "REDACTED FOR PRIVACY",
-          "REDACTED FOR PRIVACY",
-          "XX"
-        ]
-      ],
-      ["tel",{"type":["voice"]},"uri","tel:REDACTED FOR PRIVACY"],
-      ["tel",{"type":["fax"]},"uri","tel:REDACTED FOR PRIVACY"]
     ]
   ],
   "remarks": [

core/src/test/resources/google/registry/rdap/rdap_domain_no_contacts_with_remark.json

@@ -150,7 +150,7 @@
     },
     {
       "objectClassName": "entity",
-      "handle": "REDACTED FOR PRIVACY",
+      "handle": "",
       "roles":["administrative"],
       "remarks": [
         {
@@ -180,18 +180,14 @@
         "vcard",
         [
           ["version", {}, "text", "4.0"],
-          ["fn", {}, "text", "REDACTED FOR PRIVACY"],
+          ["fn", {}, "text", ""]
-          ["org", {}, "text", "REDACTED FOR PRIVACY"],
-          ["adr", {}, "text", ["", "", "REDACTED FOR PRIVACY", "REDACTED FOR PRIVACY", "REDACTED FOR PRIVACY", "REDACTED FOR PRIVACY", "XX"]],
-          ["tel", {"type":["voice"]}, "uri", "tel:REDACTED FOR PRIVACY"],
-          ["tel", {"type":["fax"]}, "uri", "tel:REDACTED FOR PRIVACY"]
         ]
       ]
     },
 
     {
       "objectClassName":"entity",
-      "handle":"REDACTED FOR PRIVACY",
+      "handle":"",
       "remarks":[
         {
           "title":"REDACTED FOR PRIVACY",
@@ -221,18 +217,14 @@
         "vcard",
         [
           ["version", {}, "text", "4.0"],
-          ["fn", {}, "text", "REDACTED FOR PRIVACY"],
+          ["fn", {}, "text", ""]
-          ["org", {}, "text", "REDACTED FOR PRIVACY"],
-          ["adr", {}, "text", ["", "", "REDACTED FOR PRIVACY", "REDACTED FOR PRIVACY", "REDACTED FOR PRIVACY", "REDACTED FOR PRIVACY", "XX"]],
-          ["tel", {"type":["voice"]}, "uri", "tel:REDACTED FOR PRIVACY"],
-          ["tel", {"type":["fax"]}, "uri", "tel:REDACTED FOR PRIVACY"]
         ]
       ]
     },
 
     {
       "objectClassName":"entity",
-      "handle":"REDACTED FOR PRIVACY",
+      "handle":"",
       "remarks":[
         {
           "title":"REDACTED FOR PRIVACY",
@@ -262,11 +254,7 @@
         "vcard",
         [
           ["version", {}, "text", "4.0"],
-          ["fn", {}, "text", "REDACTED FOR PRIVACY"],
+          ["fn", {}, "text", ""]
-          ["org", {}, "text", "REDACTED FOR PRIVACY"],
-          ["adr", {}, "text", ["", "", "REDACTED FOR PRIVACY", "REDACTED FOR PRIVACY", "REDACTED FOR PRIVACY", "REDACTED FOR PRIVACY", "XX"]],
-          ["tel", {"type":["voice"]}, "uri", "tel:REDACTED FOR PRIVACY"],
-          ["tel", {"type":["fax"]}, "uri", "tel:REDACTED FOR PRIVACY"]
         ]
       ]
     }

core/src/test/resources/google/registry/rdap/rdap_domain_unicode_no_contacts_with_remark.json

@@ -153,7 +153,7 @@
     },
     {
       "objectClassName": "entity",
-      "handle": "REDACTED FOR PRIVACY",
+      "handle": "",
       "roles":["administrative"],
       "remarks": [
         {
@@ -183,18 +183,14 @@
         "vcard",
         [
           ["version", {}, "text", "4.0"],
-          ["fn", {}, "text", "REDACTED FOR PRIVACY"],
+          ["fn", {}, "text", ""]
-          ["org", {}, "text", "REDACTED FOR PRIVACY"],
-          ["adr", {}, "text", ["", "", "REDACTED FOR PRIVACY", "REDACTED FOR PRIVACY", "REDACTED FOR PRIVACY", "REDACTED FOR PRIVACY", "XX"]],
-          ["tel", {"type":["voice"]}, "uri", "tel:REDACTED FOR PRIVACY"],
-          ["tel", {"type":["fax"]}, "uri", "tel:REDACTED FOR PRIVACY"]
         ]
       ]
     },
 
     {
       "objectClassName":"entity",
-      "handle":"REDACTED FOR PRIVACY",
+      "handle":"",
       "remarks":[
         {
           "title":"REDACTED FOR PRIVACY",
@@ -224,18 +220,13 @@
         "vcard",
         [
           ["version", {}, "text", "4.0"],
-          ["fn", {}, "text", "REDACTED FOR PRIVACY"],
+          ["fn", {}, "text", ""]        ]
-          ["org", {}, "text", "REDACTED FOR PRIVACY"],
-          ["adr", {}, "text", ["", "", "REDACTED FOR PRIVACY", "REDACTED FOR PRIVACY", "REDACTED FOR PRIVACY", "REDACTED FOR PRIVACY", "XX"]],
-          ["tel", {"type":["voice"]}, "uri", "tel:REDACTED FOR PRIVACY"],
-          ["tel", {"type":["fax"]}, "uri", "tel:REDACTED FOR PRIVACY"]
-        ]
       ]
     },
 
     {
       "objectClassName":"entity",
-      "handle":"REDACTED FOR PRIVACY",
+      "handle":"",
       "remarks":[
         {
           "title":"REDACTED FOR PRIVACY",
@@ -265,11 +256,7 @@
         "vcard",
         [
           ["version", {}, "text", "4.0"],
-          ["fn", {}, "text", "REDACTED FOR PRIVACY"],
+          ["fn", {}, "text", ""]
-          ["org", {}, "text", "REDACTED FOR PRIVACY"],
-          ["adr", {}, "text", ["", "", "REDACTED FOR PRIVACY", "REDACTED FOR PRIVACY", "REDACTED FOR PRIVACY", "REDACTED FOR PRIVACY", "XX"]],
-          ["tel", {"type":["voice"]}, "uri", "tel:REDACTED FOR PRIVACY"],
-          ["tel", {"type":["fax"]}, "uri", "tel:REDACTED FOR PRIVACY"]
         ]
       ]
     }

core/src/test/resources/google/registry/rdap/rdapjson_domain_logged_out.json

@@ -137,7 +137,7 @@
     },
     {
       "objectClassName": "entity",
-      "handle": "REDACTED FOR PRIVACY",
+      "handle": "",
       "roles": ["administrative"],
       "remarks": [
         {
@@ -166,31 +166,14 @@
       "vcardArray": [
         "vcard",
         [
-          ["version",{},"text","4.0"],
+          ["version", {}, "text", "4.0"],
-          ["fn",{},"text","REDACTED FOR PRIVACY"],
+          ["fn", {}, "text", ""]
-          ["org",{},"text","REDACTED FOR PRIVACY"],
-          [
-            "adr",
-            {},
-            "text",
-            [
-              "",
-              "",
-              "REDACTED FOR PRIVACY",
-              "REDACTED FOR PRIVACY",
-              "REDACTED FOR PRIVACY",
-              "REDACTED FOR PRIVACY",
-              "XX"
-            ]
-          ],
-          ["tel",{"type":["voice"]},"uri","tel:REDACTED FOR PRIVACY"],
-          ["tel",{"type":["fax"]},"uri","tel:REDACTED FOR PRIVACY"]
         ]
       ]
     },
     {
       "objectClassName": "entity",
-      "handle": "REDACTED FOR PRIVACY",
+      "handle": "",
       "roles": ["technical"],
       "remarks": [
         {
@@ -219,31 +202,14 @@
       "vcardArray": [
         "vcard",
         [
-          ["version",{},"text","4.0"],
+          ["version", {}, "text", "4.0"],
-          ["fn",{},"text","REDACTED FOR PRIVACY"],
+          ["fn", {}, "text", ""]
-          ["org",{},"text","REDACTED FOR PRIVACY"],
-          [
-            "adr",
-            {},
-            "text",
-            [
-              "",
-              "",
-              "REDACTED FOR PRIVACY",
-              "REDACTED FOR PRIVACY",
-              "REDACTED FOR PRIVACY",
-              "REDACTED FOR PRIVACY",
-              "XX"
-            ]
-          ],
-          ["tel",{"type":["voice"]},"uri","tel:REDACTED FOR PRIVACY"],
-          ["tel",{"type":["fax"]},"uri","tel:REDACTED FOR PRIVACY"]
         ]
       ]
     },
     {
       "objectClassName": "entity",
-      "handle": "REDACTED FOR PRIVACY",
+      "handle": "",
       "roles": ["registrant"],
       "remarks": [
         {
@@ -272,25 +238,8 @@
       "vcardArray": [
         "vcard",
         [
-          ["version",{},"text","4.0"],
+          ["version", {}, "text", "4.0"],
-          ["fn",{},"text","REDACTED FOR PRIVACY"],
+          ["fn", {}, "text", ""]
-          ["org",{},"text","REDACTED FOR PRIVACY"],
-          [
-            "adr",
-            {},
-            "text",
-            [
-              "",
-              "",
-              "REDACTED FOR PRIVACY",
-              "REDACTED FOR PRIVACY",
-              "REDACTED FOR PRIVACY",
-              "REDACTED FOR PRIVACY",
-              "XX"
-            ]
-          ],
-          ["tel",{"type":["voice"]},"uri","tel:REDACTED FOR PRIVACY"],
-          ["tel",{"type":["fax"]},"uri","tel:REDACTED FOR PRIVACY"]
         ]
       ]
     }

core/src/test/resources/google/registry/rdap/rdapjson_registrant_logged_out.json

@@ -1,6 +1,6 @@
 {
   "objectClassName" : "entity",
-  "handle" : "REDACTED FOR PRIVACY",
+  "handle" : "",
   "roles" : ["registrant"],
   "events": [
     {
@@ -11,25 +11,8 @@
   "vcardArray": [
     "vcard",
     [
-      ["version",{},"text","4.0"],
+      ["version", {}, "text", "4.0"],
-      ["fn",{},"text","REDACTED FOR PRIVACY"],
+      ["fn", {}, "text", ""]
-      ["org",{},"text","REDACTED FOR PRIVACY"],
-      [
-        "adr",
-        {},
-        "text",
-        [
-          "",
-          "",
-          "REDACTED FOR PRIVACY",
-          "REDACTED FOR PRIVACY",
-          "REDACTED FOR PRIVACY",
-          "REDACTED FOR PRIVACY",
-          "XX"
-        ]
-      ],
-      ["tel",{"type":["voice"]},"uri","tel:REDACTED FOR PRIVACY"],
-      ["tel",{"type":["fax"]},"uri","tel:REDACTED FOR PRIVACY"]
     ]
   ],
   "remarks": [