Create separate BSA service (#2221)

This commit is contained in:
Pavlo Tkach 2023-11-15 18:38:26 -05:00 committed by GitHub
parent 445825957d
commit 572b7101cb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
31 changed files with 559 additions and 28 deletions

View file

@ -347,6 +347,7 @@ subprojects {
def services = [':services:default',
':services:backend',
':services:bsa',
':services:tools',
':services:pubapi']

View file

@ -0,0 +1,45 @@
// Copyright 2023 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.bsa;
import static javax.servlet.http.HttpServletResponse.SC_OK;
import google.registry.request.Action;
import google.registry.request.Action.Service;
import google.registry.request.Response;
import google.registry.request.auth.Auth;
import javax.inject.Inject;
@Action(
service = Service.BSA,
path = PlaceholderAction.PATH,
method = Action.Method.GET,
auth = Auth.AUTH_API_ADMIN)
public class PlaceholderAction implements Runnable {
private final Response response;
static final String PATH = "/_dr/task/bsaDownload";
@Inject
public PlaceholderAction(Response response) {
this.response = response;
}
@Override
public void run() {
response.setStatus(SC_OK);
response.setPayload("Hello World");
}
}

View file

@ -1458,6 +1458,15 @@ public final class RegistryConfig {
return makeUrl(CONFIG_SETTINGS.get().gcpProject.backendServiceUrl);
}
/**
* Returns the address of the Nomulus app bsa HTTP server.
*
* <p>This is used by the {@code nomulus} tool to connect to the App Engine remote API.
*/
public static URL getBsaServer() {
return makeUrl(CONFIG_SETTINGS.get().gcpProject.bsaServiceUrl);
}
/**
* Returns the address of the Nomulus app tools HTTP server.
*

View file

@ -53,6 +53,7 @@ public class RegistryConfigSettings {
public boolean isLocal;
public String defaultServiceUrl;
public String backendServiceUrl;
public String bsaServiceUrl;
public String toolsServiceUrl;
public String pubapiServiceUrl;
}

View file

@ -20,9 +20,11 @@ gcpProject:
# URLs of the services for the project.
defaultServiceUrl: https://default.example.com
backendServiceUrl: https://backend.example.com
bsaServiceUrl: https://bsa.example.com
toolsServiceUrl: https://tools.example.com
pubapiServiceUrl: https://pubapi.example.com
gSuite:
# Publicly accessible domain name of the running G Suite instance.
domainName: domain-registry.example

View file

@ -18,13 +18,6 @@
value="alpha"/>
</system-properties>
<!-- Enable external traffic to go through VPC, required for static ip -->
<vpc-access-connector>
<name>projects/domain-registry-alpha/locations/us-central1/connectors/appengine-connector</name>
<egress-setting>all-traffic</egress-setting>
</vpc-access-connector>
<static-files>
<include path="/*.html" expiration="1m"/>
</static-files>

View file

@ -0,0 +1,31 @@
<?xml version="1.0" encoding="UTF-8"?>
<appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
<runtime>java17</runtime>
<service>bsa</service>
<app-engine-apis>true</app-engine-apis>
<sessions-enabled>true</sessions-enabled>
<instance-class>B4</instance-class>
<basic-scaling>
<max-instances>100</max-instances>
<idle-timeout>10m</idle-timeout>
</basic-scaling>
<system-properties>
<property name="java.util.logging.config.file"
value="WEB-INF/logging.properties"/>
<property name="google.registry.environment"
value="alpha"/>
</system-properties>
<!-- Enable external traffic to go through VPC, required for static ip -->
<vpc-access-connector>
<name>projects/domain-registry-alpha/locations/us-central1/connectors/appengine-connector</name>
<egress-setting>all-traffic</egress-setting>
</vpc-access-connector>
<static-files>
<include path="/*.html" expiration="1m"/>
</static-files>
</appengine-web-app>

View file

@ -31,6 +31,12 @@ encoding="UTF-8"?>
<context-root>backend</context-root>
</web>
</module>
<module>
<web>
<web-uri>bsa</web-uri>
<context-root>bsa</context-root>
</web>
</module>
<module>
<web>
<web-uri>tools</web-uri>

View file

@ -0,0 +1,17 @@
# A default java.util.logging configuration.
# (All App Engine logging is through java.util.logging by default).
#
# To use this configuration, copy it into your application's WEB-INF
# folder and add the following to your appengine-web.xml:
#
# <system-properties>
# <property name="java.util.logging.config.file" value="WEB-INF/logging.properties"/>
# </system-properties>
#
# Set the default logging level for all loggers to INFO.
.level = INFO
# Turn off logging in Hibernate classes for misleading ERROR-level logs
org.hibernate.engine.jdbc.batch.internal.BatchingBatch.level=OFF
org.hibernate.engine.jdbc.spi.SqlExceptionHelper.level=OFF

View file

@ -0,0 +1,70 @@
<?xml version="1.0" encoding="UTF-8"?>
<web-app xmlns="http://java.sun.com/xml/ns/javaee" version="2.5"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://java.sun.com/xml/ns/javaee
http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd">
<!-- Servlets -->
<!-- Servlet for injected backends actions -->
<servlet>
<display-name>BsaServlet</display-name>
<servlet-name>bsa-servlet</servlet-name>
<servlet-class>google.registry.module.bsa.BsaServlet</servlet-class>
<load-on-startup>1</load-on-startup>
</servlet>
<!-- Test action -->
<servlet-mapping>
<servlet-name>backend-servlet</servlet-name>
<url-pattern>/_dr/task/bsa</url-pattern>
</servlet-mapping>
<!-- Security config -->
<security-constraint>
<web-resource-collection>
<web-resource-name>Internal</web-resource-name>
<description>
Admin-only internal section. Requests for paths covered by the URL patterns below will be
checked for a logged-in user account that's allowed to access the AppEngine admin console
(NOTE: this includes Editor/Viewer permissions in addition to Owner and the new IAM
App Engine Admin role. See https://cloud.google.com/appengine/docs/java/access-control
specifically the "Access handlers that have a login:admin restriction" line.)
TODO(b/28219927): lift some of these restrictions so that we can allow OAuth authentication
for endpoints that need to be accessed by open-source automated processes.
</description>
<!-- Internal AppEngine endpoints. The '_ah' is short for app hosting. -->
<url-pattern>/_ah/*</url-pattern>
<!-- Registrar console (should not be available on non-default module). -->
<url-pattern>/registrar*</url-pattern>
<!-- Verbatim JavaScript sources (only visible to admins for debugging). -->
<url-pattern>/assets/sources/*</url-pattern>
</web-resource-collection>
<auth-constraint>
<role-name>admin</role-name>
</auth-constraint>
<!-- Repeated here since catch-all rule below is not inherited. -->
<user-data-constraint>
<transport-guarantee>CONFIDENTIAL</transport-guarantee>
</user-data-constraint>
</security-constraint>
<!-- Require TLS on all requests. -->
<security-constraint>
<web-resource-collection>
<web-resource-name>Secure</web-resource-name>
<description>
Require encryption for all paths. http URLs will be redirected to https.
</description>
<url-pattern>/*</url-pattern>
</web-resource-collection>
<user-data-constraint>
<transport-guarantee>CONFIDENTIAL</transport-guarantee>
</user-data-constraint>
</security-constraint>
</web-app>

View file

@ -18,12 +18,6 @@
value="crash"/>
</system-properties>
<!-- Enable external traffic to go through VPC, required for static ip -->
<vpc-access-connector>
<name>projects/domain-registry-crash/locations/us-central1/connectors/appengine-connector</name>
<egress-setting>all-traffic</egress-setting>
</vpc-access-connector>
<static-files>
<include path="/*.html" expiration="1m"/>
</static-files>

View file

@ -0,0 +1,30 @@
<?xml version="1.0" encoding="UTF-8"?>
<appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
<runtime>java17</runtime>
<service>bsa</service>
<app-engine-apis>true</app-engine-apis>
<sessions-enabled>true</sessions-enabled>
<instance-class>B4</instance-class>
<basic-scaling>
<max-instances>10</max-instances>
<idle-timeout>10m</idle-timeout>
</basic-scaling>
<system-properties>
<property name="java.util.logging.config.file"
value="WEB-INF/logging.properties"/>
<property name="google.registry.environment"
value="crash"/>
</system-properties>
<!-- Enable external traffic to go through VPC, required for static ip -->
<vpc-access-connector>
<name>projects/domain-registry-crash/locations/us-central1/connectors/appengine-connector</name>
<egress-setting>all-traffic</egress-setting>
</vpc-access-connector>
<static-files>
<include path="/*.html" expiration="1m"/>
</static-files>
</appengine-web-app>

View file

@ -0,0 +1,28 @@
<?xml version="1.0" encoding="UTF-8"?>
<appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
<runtime>java17</runtime>
<service>bsa</service>
<app-engine-apis>true</app-engine-apis>
<sessions-enabled>true</sessions-enabled>
<instance-class>B4</instance-class>
<basic-scaling>
<max-instances>10</max-instances>
<idle-timeout>10m</idle-timeout>
</basic-scaling>
<system-properties>
<property name="java.util.logging.config.file"
value="WEB-INF/logging.properties"/>
<property name="google.registry.environment"
value="local"/>
<property name="appengine.generated.dir"
value="/tmp/domain-registry-appengine-generated/local/"/>
</system-properties>
<static-files>
<include path="/*.html">
<http-header name="Cache-Control" value="max-age=0,must-revalidate" />
</include>
</static-files>
</appengine-web-app>

View file

@ -0,0 +1,30 @@
<?xml version="1.0" encoding="UTF-8"?>
<appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
<runtime>java8</runtime>
<service>bsa</service>
<!--app-engine-apis>true</app-engine-apis-->
<threadsafe>true</threadsafe>
<sessions-enabled>true</sessions-enabled>
<instance-class>B4_1G</instance-class>
<basic-scaling>
<max-instances>100</max-instances>
<idle-timeout>10m</idle-timeout>
</basic-scaling>
<system-properties>
<property name="java.util.logging.config.file"
value="WEB-INF/logging.properties"/>
<property name="google.registry.environment"
value="production"/>
</system-properties>
<static-files>
<include path="/*.html" expiration="1d"/>
</static-files>
<!-- Prevent uncaught servlet errors from leaking a stack trace. -->
<static-error-handlers>
<handler file="error.html"/>
</static-error-handlers>
</appengine-web-app>

View file

@ -22,12 +22,6 @@
<include path="/*.html" expiration="1h"/>
</static-files>
<!-- Enable external traffic to go through VPC, required for static ip -->
<vpc-access-connector>
<name>projects/domain-registry-qa/locations/us-central1/connectors/appengine-connector</name>
<egress-setting>all-traffic</egress-setting>
</vpc-access-connector>
<!-- Prevent uncaught servlet errors from leaking a stack trace. -->
<static-error-handlers>
<handler file="error.html"/>

View file

@ -0,0 +1,35 @@
<?xml version="1.0" encoding="UTF-8"?>
<appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
<runtime>java17</runtime>
<service>bsa</service>
<app-engine-apis>true</app-engine-apis>
<sessions-enabled>true</sessions-enabled>
<instance-class>B4</instance-class>
<basic-scaling>
<max-instances>10</max-instances>
<idle-timeout>10m</idle-timeout>
</basic-scaling>
<system-properties>
<property name="java.util.logging.config.file"
value="WEB-INF/logging.properties"/>
<property name="google.registry.environment"
value="qa"/>
</system-properties>
<static-files>
<include path="/*.html" expiration="1h"/>
</static-files>
<!-- Enable external traffic to go through VPC, required for static ip -->
<vpc-access-connector>
<name>projects/domain-registry-qa/locations/us-central1/connectors/appengine-connector</name>
<egress-setting>all-traffic</egress-setting>
</vpc-access-connector>
<!-- Prevent uncaught servlet errors from leaking a stack trace. -->
<static-error-handlers>
<handler file="error.html"/>
</static-error-handlers>
</appengine-web-app>

View file

@ -23,12 +23,6 @@
<include path="/*.html" expiration="1d"/>
</static-files>
<!-- Enable external traffic to go through VPC, required for static ip -->
<vpc-access-connector>
<name>projects/domain-registry-sandbox/locations/us-central1/connectors/appengine-connector</name>
<egress-setting>all-traffic</egress-setting>
</vpc-access-connector>
<!-- Prevent uncaught servlet errors from leaking a stack trace. -->
<static-error-handlers>
<handler file="error.html"/>

View file

@ -0,0 +1,36 @@
<?xml version="1.0" encoding="UTF-8"?>
<appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
<runtime>java8</runtime>
<service>bsa</service>
<!--app-engine-apis>true</app-engine-apis-->
<threadsafe>true</threadsafe>
<sessions-enabled>true</sessions-enabled>
<instance-class>B4</instance-class>
<basic-scaling>
<max-instances>100</max-instances>
<idle-timeout>10m</idle-timeout>
</basic-scaling>
<system-properties>
<property name="java.util.logging.config.file"
value="WEB-INF/logging.properties"/>
<property name="google.registry.environment"
value="sandbox"/>
</system-properties>
<static-files>
<include path="/*.html" expiration="1d"/>
</static-files>
<!-- Enable external traffic to go through VPC, required for static ip -->
<vpc-access-connector>
<name>projects/domain-registry-sandbox/locations/us-central1/connectors/appengine-connector</name>
<egress-setting>all-traffic</egress-setting>
</vpc-access-connector>
<!-- Prevent uncaught servlet errors from leaking a stack trace. -->
<static-error-handlers>
<handler file="error.html"/>
</static-error-handlers>
</appengine-web-app>

View file

@ -0,0 +1,46 @@
// Copyright 2023 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.module.bsa;
import com.google.monitoring.metrics.MetricReporter;
import dagger.Component;
import dagger.Lazy;
import google.registry.config.CredentialModule;
import google.registry.config.RegistryConfig.ConfigModule;
import google.registry.module.bsa.BsaRequestComponent.BsaRequestComponentModule;
import google.registry.monitoring.whitebox.StackdriverModule;
import google.registry.request.Modules.GsonModule;
import google.registry.request.Modules.UserServiceModule;
import google.registry.request.auth.AuthModule;
import google.registry.util.UtilsModule;
import javax.inject.Singleton;
@Singleton
@Component(
modules = {
AuthModule.class,
UtilsModule.class,
UserServiceModule.class,
GsonModule.class,
ConfigModule.class,
StackdriverModule.class,
CredentialModule.class,
BsaRequestComponentModule.class
})
interface BsaComponent {
BsaRequestHandler requestHandler();
Lazy<MetricReporter> metricReporter();
}

View file

@ -0,0 +1,45 @@
// Copyright 2023 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.module.bsa;
import dagger.Module;
import dagger.Subcomponent;
import google.registry.bsa.PlaceholderAction;
import google.registry.request.RequestComponentBuilder;
import google.registry.request.RequestModule;
import google.registry.request.RequestScope;
@RequestScope
@Subcomponent(
modules = {
RequestModule.class,
})
interface BsaRequestComponent {
PlaceholderAction bsaAction();
@Subcomponent.Builder
abstract class Builder implements RequestComponentBuilder<BsaRequestComponent> {
@Override
public abstract Builder requestModule(RequestModule requestModule);
@Override
public abstract BsaRequestComponent build();
}
@Module(subcomponents = BsaRequestComponent.class)
class BsaRequestComponentModule {}
}

View file

@ -0,0 +1,29 @@
// Copyright 2023 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.module.bsa;
import google.registry.request.RequestHandler;
import google.registry.request.auth.RequestAuthenticator;
import javax.inject.Inject;
import javax.inject.Provider;
public class BsaRequestHandler extends RequestHandler<BsaRequestComponent> {
@Inject
public BsaRequestHandler(
Provider<BsaRequestComponent.Builder> componentBuilderProvider,
RequestAuthenticator requestAuthenticator) {
super(componentBuilderProvider, requestAuthenticator);
}
}

View file

@ -0,0 +1,30 @@
// Copyright 2023 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.module.bsa;
import com.google.monitoring.metrics.MetricReporter;
import dagger.Lazy;
import google.registry.module.ServletBase;
public final class BsaServlet extends ServletBase {
private static final BsaComponent component = DaggerBsaComponent.create();
private static final BsaRequestHandler requestHandler = component.requestHandler();
private static final Lazy<MetricReporter> metricReporter = component.metricReporter();
public BsaServlet() {
super(requestHandler, metricReporter);
}
}

View file

@ -0,0 +1,16 @@
// Copyright 2023 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
@javax.annotation.ParametersAreNonnullByDefault
package google.registry.module.bsa;

View file

@ -30,11 +30,13 @@ public @interface Action {
/** App Engine services supported by the request processor. */
enum Service {
BSA("bsa"),
DEFAULT("default"),
TOOLS("tools"),
BACKEND("backend"),
PUBAPI("pubapi");
private final String serviceId;
Service(String serviceId) {

View file

@ -170,6 +170,8 @@ public class ServiceConnection {
return RegistryConfig.getToolsServer();
case BACKEND:
return RegistryConfig.getBackendServer();
case BSA:
return RegistryConfig.getBsaServer();
case PUBAPI:
return RegistryConfig.getPubapiServer();
}

View file

@ -0,0 +1,36 @@
// Copyright 2023 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.module.bsa;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.junit.jupiter.api.Test;
class BsaServletTest {
private final HttpServletRequest req = mock(HttpServletRequest.class);
private final HttpServletResponse rsp = mock(HttpServletResponse.class);
@Test
void testService_unknownPath_returnsNotFound() throws Exception {
when(req.getMethod()).thenReturn("GET");
when(req.getRequestURI()).thenReturn("/lol");
new BsaServlet().service(req, rsp);
verify(rsp).sendError(404);
}
}

View file

@ -66,6 +66,14 @@ sized to support not just the normal ongoing DNS load but also the load incurred
by MapReduces, both scheduled (such as RDE) and on-demand (asynchronous
contact/host deletion).
#### BSA service
The bsa service is responsible for business logic behind Nomulus and BSA
functionality. Requests to the backend service are handled by the `BsaServlet`,
which provides all of the endpoints exposed in `BsaRequestComponent`. These
include tasks for downloading, processing and uploading BSA data.
#### Tools service
The tools service is responsible for servicing requests from the `nomulus`

View file

@ -46,7 +46,7 @@ else
-PmavenUrl="${gcs_prefix}"/maven \
-PpluginsUrl="${gcs_prefix}"/plugins
for service in default pubapi backend tools
for service in default pubapi backend bsa tools
do
mv services/"${service}"/build/staged-app "${dest}/${service}"
done

View file

@ -43,7 +43,7 @@ steps:
gcloud auth activate-service-account --key-file=tool-credential.json
for service in default pubapi backend tools
for service in default pubapi backend bsa tools
do
for version in $(gcloud app versions list \
--filter="SERVICE:$service AND SERVING_STATUS:STOPPED" \

View file

@ -75,7 +75,7 @@ steps:
gcloud app versions list \
--project $project_id --hide-no-traffic \
--format="csv[no-heading](SERVICE,VERSION.ID)" | \
grep -e "^backend\|^default\|^pubapi\|^tools" |\
grep -e "^backend\|^default\|^bsa\|^pubapi\|^tools" |\
while read line; do echo "${TAG_NAME},$line"; done | tee "$local_map"
num_versions=$(cat "$local_map" | wc -l)
if [ "$num_versions" -ne 4 ]; then

View file

@ -43,6 +43,7 @@ include 'proxy'
include 'util'
include 'services:default'
include 'services:backend'
include 'services:bsa'
include 'services:tools'
include 'services:pubapi'
include 'java8compatibility'