diff --git a/cdap-api-common/pom.xml b/cdap-api-common/pom.xml index 4096488d6697..2ac898352e5b 100644 --- a/cdap-api-common/pom.xml +++ b/cdap-api-common/pom.xml @@ -45,4 +45,18 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 17 + 17 + + + + + diff --git a/cdap-api-spark3_2.12/pom.xml b/cdap-api-spark3_2.12/pom.xml index badbb8132e43..733ea9b56a75 100644 --- a/cdap-api-spark3_2.12/pom.xml +++ b/cdap-api-spark3_2.12/pom.xml @@ -187,6 +187,15 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + net.alchim31.maven scala-maven-plugin diff --git a/cdap-api/pom.xml b/cdap-api/pom.xml index 26e9a4ab3c03..60c807201e9b 100644 --- a/cdap-api/pom.xml +++ b/cdap-api/pom.xml @@ -81,5 +81,16 @@ true + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 17 + 17 + + + diff --git a/cdap-app-fabric-tests/pom.xml b/cdap-app-fabric-tests/pom.xml index a258bcb191b0..8fc7a35c28a0 100644 --- a/cdap-app-fabric-tests/pom.xml +++ b/cdap-app-fabric-tests/pom.xml @@ -132,6 +132,15 @@ the License. + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + org.sonatype.central diff --git a/cdap-app-fabric/pom.xml b/cdap-app-fabric/pom.xml index 48b08feacaf0..5bce367b68fe 100644 --- a/cdap-app-fabric/pom.xml +++ b/cdap-app-fabric/pom.xml @@ -274,10 +274,18 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 17 + 17 + + org.apache.maven.plugins maven-jar-plugin - 2.4 test-jar diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/DefaultPreviewRunnerManager.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/DefaultPreviewRunnerManager.java index d8f5829b1356..ed72f70e6a80 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/DefaultPreviewRunnerManager.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/DefaultPreviewRunnerManager.java @@ -124,12 +124,12 @@ protected void startUp() throws Exception { // Starts common services runner = previewInjector.getInstance(PreviewRunner.class); if (runner instanceof Service) { - ((Service) runner).startAndWait(); + ((Service) runner).startAsync().awaitRunning(); } // Create and start the preview poller services. for (int i = 0; i < maxConcurrentPreviews; i++) { - createPreviewRunnerService().startAndWait(); + createPreviewRunnerService().startAsync().awaitRunning(); } } @@ -144,7 +144,7 @@ protected void shutDown() throws Exception { private void stopQuietly(Service service) { try { - service.stopAndWait(); + service.stopAsync().awaitTerminated(); } catch (Exception e) { LOG.warn("Error stopping the preview runner.", e); } @@ -163,8 +163,8 @@ public void stop(ApplicationId preview) throws Exception { } PreviewRunnerService newRunnerService = createPreviewRunnerService(); - runnerService.stopAndWait(); - newRunnerService.startAndWait(); + runnerService.stopAsync().awaitTerminated(); + newRunnerService.startAsync().awaitRunning(); } @Override @@ -237,14 +237,14 @@ public InetAddress providesHostname(CConfiguration cConf) { private PreviewRunnerService createPreviewRunnerService() { PreviewRunnerService previewRunnerService = previewRunnerServiceFactory.create(runner); - previewRunnerService.addListener(new ServiceListenerAdapter() { + previewRunnerService.addListener(new Service.Listener() { @Override - public void terminated(State from) { + public void terminated(Service.State from) { previewRunnerServices.remove(previewRunnerService); if (previewRunnerServices.isEmpty()) { try { - stop(); + stopAsync(); } catch (Exception e) { // should not happen LOG.error("Failed to shutdown the preview runner manager service.", e); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewHttpServer.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewHttpServer.java index d93067bb3668..0433d5beb02f 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewHttpServer.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/PreviewHttpServer.java @@ -101,7 +101,7 @@ protected void startUp() throws Exception { Constants.Logging.COMPONENT_NAME, Constants.Service.PREVIEW_HTTP)); if (previewManager instanceof Service) { - ((Service) previewManager).startAndWait(); + ((Service) previewManager).startAsync().awaitRunning(); } httpService.start(); @@ -117,7 +117,7 @@ protected void shutDown() throws Exception { try { cancelHttpService.cancel(); if (previewManager instanceof Service) { - ((Service) previewManager).stopAndWait(); + ((Service) previewManager).stopAsync().awaitTerminated(); } } finally { httpService.stop(); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/runtime/AbstractProgramRuntimeService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/runtime/AbstractProgramRuntimeService.java index d664d16171ff..e5dc6a329ce8 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/runtime/AbstractProgramRuntimeService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/runtime/AbstractProgramRuntimeService.java @@ -43,6 +43,7 @@ import io.cdap.cdap.proto.id.ProgramId; import io.cdap.cdap.proto.id.ProgramRunId; import java.io.Closeable; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.EnumSet; @@ -472,7 +473,11 @@ private ProgramController createController(ProgramId programId, RunId runId, */ private void cleanupRuntimeInfo(@Nullable RuntimeInfo info) { if (info instanceof Closeable) { - Closeables.closeQuietly((Closeable) info); + try { + ((Closeable) info).close(); + } catch (IOException e) { + // Ignore + } } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/runtime/DelayedProgramController.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/runtime/DelayedProgramController.java index 0e8f402aab22..48e9cc9f0877 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/runtime/DelayedProgramController.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/runtime/DelayedProgramController.java @@ -189,7 +189,7 @@ public void onSuccess(ProgramController result) { public void onFailure(Throwable t) { resultFuture.setException(t); } - }); + }, com.google.common.util.concurrent.MoreExecutors.directExecutor()); }); return resultFuture; } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/common/twill/TwillAppLifecycleEventHandler.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/common/twill/TwillAppLifecycleEventHandler.java index 4d5342a5882a..169e8c68b402 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/common/twill/TwillAppLifecycleEventHandler.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/common/twill/TwillAppLifecycleEventHandler.java @@ -124,7 +124,7 @@ public void initialize(EventHandlerContext context) { if (clusterMode == ClusterMode.ON_PREMISE) { zkClientService = injector.getInstance(ZKClientService.class); - zkClientService.startAndWait(); + zkClientService.startAsync().awaitRunning(); } LoggingContextAccessor.setLoggingContext( @@ -210,9 +210,15 @@ public void aborted() { @Override public void destroy() { - Closeables.closeQuietly(logAppenderInitializer); + try { + if (logAppenderInitializer != null) { + logAppenderInitializer.close(); + } + } catch (Exception e) { + // Ignore + } if (zkClientService != null) { - zkClientService.stop(); + zkClientService.stopAsync(); } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AuthorizationHandler.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AuthorizationHandler.java index af6697eeb6ff..6a67f4ba87f6 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AuthorizationHandler.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/AuthorizationHandler.java @@ -16,7 +16,7 @@ package io.cdap.cdap.gateway.handlers; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.reflect.TypeToken; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -326,7 +326,7 @@ private void ensureSecurityEnabled() throws FeatureDisabledException { private void createLogEntry(HttpRequest httpRequest, HttpResponseStatus responseStatus) throws UnknownHostException { InetAddress clientAddr = InetAddress.getByName( - Objects.firstNonNull(SecurityRequestContext.getUserIp(), "0.0.0.0")); + MoreObjects.firstNonNull(SecurityRequestContext.getUserIp(), "0.0.0.0")); AuditLogEntry logEntry = new AuditLogEntry(httpRequest, clientAddr.getHostAddress()); logEntry.setUserName(authenticationContext.getPrincipal().getName()); logEntry.setResponse(responseStatus.code(), 0L); @@ -334,7 +334,7 @@ private void createLogEntry(HttpRequest httpRequest, HttpResponseStatus response } private Set getRequestPermissions(AuthorizationRequest request) { - Set permissions = Objects.firstNonNull(request.getPermissions(), + Set permissions = MoreObjects.firstNonNull(request.getPermissions(), Collections.emptySet()); if (request.getActions() != null) { permissions = Stream.concat(permissions.stream(), diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/DatasetServiceStore.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/DatasetServiceStore.java index 795e8d771e93..e60ff0176028 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/DatasetServiceStore.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/DatasetServiceStore.java @@ -17,9 +17,9 @@ package io.cdap.cdap.gateway.handlers; import com.google.common.base.Preconditions; -import com.google.common.collect.DiscreteDomains; +import com.google.common.collect.DiscreteDomain; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Ranges; +import com.google.common.collect.Range; import com.google.common.util.concurrent.AbstractIdleService; import com.google.gson.Gson; import com.google.inject.Inject; @@ -120,8 +120,8 @@ public synchronized void setRestartAllInstancesRequest(String serviceName, long RestartStatus status = isSuccess ? RestartStatus.SUCCESS : RestartStatus.FAILURE; Integer serviceInstance = getServiceInstance(serviceName); int instanceCount = (serviceInstance == null) ? 0 : serviceInstance; - Set instancesToRestart = Ranges.closedOpen(0, instanceCount) - .asSet(DiscreteDomains.integers()); + Set instancesToRestart = com.google.common.collect.ContiguousSet.create( + Range.closedOpen(0, instanceCount), DiscreteDomain.integers()); RestartServiceInstancesStatus restartStatus = new RestartServiceInstancesStatus(serviceName, startTimeMs, endTimeMs, status, diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/ProgramScheduleHttpHandler.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/ProgramScheduleHttpHandler.java index cf3028ba0c8e..0c243908077b 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/ProgramScheduleHttpHandler.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/ProgramScheduleHttpHandler.java @@ -18,7 +18,7 @@ import com.google.common.base.Charsets; import com.google.common.base.Joiner; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonSyntaxException; @@ -592,13 +592,13 @@ private void doAddSchedule(FullHttpRequest request, HttpResponder responder, Str // Schedules are versionless lifecycleService.ensureLatestProgramExists(programId.getProgramReference()); - String description = Objects.firstNonNull(scheduleFromRequest.getDescription(), ""); - Map properties = Objects.firstNonNull(scheduleFromRequest.getProperties(), + String description = MoreObjects.firstNonNull(scheduleFromRequest.getDescription(), ""); + Map properties = MoreObjects.firstNonNull(scheduleFromRequest.getProperties(), Collections.emptyMap()); - List constraints = Objects.firstNonNull( + List constraints = MoreObjects.firstNonNull( scheduleFromRequest.getConstraints(), NO_CONSTRAINTS); long timeoutMillis = - Objects.firstNonNull(scheduleFromRequest.getTimeoutMillis(), + MoreObjects.firstNonNull(scheduleFromRequest.getTimeoutMillis(), Schedulers.JOB_QUEUE_TIMEOUT_MILLIS); ProgramSchedule schedule = new ProgramSchedule(scheduleName, description, programId, properties, scheduleFromRequest.getTrigger(), constraints, timeoutMillis); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/InMemoryProgramRunDispatcher.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/InMemoryProgramRunDispatcher.java index 3bbe1bf42171..8c90e2bc8105 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/InMemoryProgramRunDispatcher.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/deploy/InMemoryProgramRunDispatcher.java @@ -297,10 +297,14 @@ public ProgramController dispatchProgram(ProgramRunDispatcherContext dispatcherC if (artifactsComputeHash && (artifactsComputeHashSnapshot || !artifactDescriptor.getArtifactId().getVersion().isSnapshot())) { Hasher hasher = Hashing.sha256().newHasher(); - hasher.putString(artifactDescriptor.getNamespace()); - hasher.putString(artifactDescriptor.getArtifactId().getName()); - hasher.putString(artifactDescriptor.getArtifactId().getScope().name()); - hasher.putString(artifactDescriptor.getArtifactId().getVersion().getVersion()); + hasher.putString(artifactDescriptor.getNamespace(), + java.nio.charset.StandardCharsets.UTF_8); + hasher.putString(artifactDescriptor.getArtifactId().getName(), + java.nio.charset.StandardCharsets.UTF_8); + hasher.putString(artifactDescriptor.getArtifactId().getScope().name(), + java.nio.charset.StandardCharsets.UTF_8); + hasher.putString(artifactDescriptor.getArtifactId().getVersion().getVersion(), + java.nio.charset.StandardCharsets.UTF_8); Map arguments = new HashMap<>(options.getArguments().asMap()); arguments.put(ProgramOptionConstants.PROGRAM_JAR_HASH, getArtifactHash(hasher)); @@ -493,7 +497,11 @@ private Runnable createCleanupTask(final Object... resources) { file.delete(); } } else if (resource instanceof Closeable) { - Closeables.closeQuietly((Closeable) resource); + try { + ((Closeable) resource).close(); + } catch (IOException e) { + // Ignore + } } else if (resource instanceof Runnable) { ((Runnable) resource).run(); } @@ -558,9 +566,9 @@ private ProgramOptions updateProgramOptions(ArtifactId artifactId, ProgramId pro } private static void hashArtifactId(Hasher hasher, ArtifactId artifactId) { - hasher.putString(artifactId.getParent().toString()); - hasher.putString(artifactId.getArtifact()); - hasher.putString(artifactId.getVersion()); + hasher.putString(artifactId.getParent().toString(), java.nio.charset.StandardCharsets.UTF_8); + hasher.putString(artifactId.getArtifact(), java.nio.charset.StandardCharsets.UTF_8); + hasher.putString(artifactId.getVersion(), java.nio.charset.StandardCharsets.UTF_8); } /** diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/DefaultPreviewManager.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/DefaultPreviewManager.java index 0f7ef6d06697..ae1c2e62e1b6 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/DefaultPreviewManager.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/DefaultPreviewManager.java @@ -147,6 +147,7 @@ public class DefaultPreviewManager extends AbstractIdleService implements Previe private final PreviewDataCleanupService previewDataCleanupService; private final MetricsCollectionService metricsCollectionService; private final AeadCipher userEncryptionAeadCipher; + private boolean metricsCollectionServiceStartedByMe = false; private Injector previewInjector; private PreviewDataSubscriberService dataSubscriberService; private PreviewTMSLogSubscriber logSubscriberService; @@ -191,7 +192,10 @@ public class DefaultPreviewManager extends AbstractIdleService implements Previe protected void startUp() throws Exception { previewInjector = createPreviewInjector(); StoreDefinition.createAllTables(previewInjector.getInstance(StructuredTableAdmin.class)); - metricsCollectionService.start(); + if (metricsCollectionService.state() == Service.State.NEW) { + metricsCollectionService.startAsync(); + metricsCollectionServiceStartedByMe = true; + } logAppender = previewInjector.getInstance(LogAppender.class); logAppender.start(); LoggingContextAccessor.setLoggingContext( @@ -199,10 +203,10 @@ protected void startUp() throws Exception { Constants.Logging.COMPONENT_NAME, Constants.Service.PREVIEW_HTTP)); logSubscriberService = previewInjector.getInstance(PreviewTMSLogSubscriber.class); - logSubscriberService.startAndWait(); + logSubscriberService.startAsync().awaitRunning(); dataSubscriberService = previewInjector.getInstance(PreviewDataSubscriberService.class); - dataSubscriberService.startAndWait(); - previewDataCleanupService.startAndWait(); + dataSubscriberService.startAsync().awaitRunning(); + previewDataCleanupService.startAsync().awaitRunning(); } @Override @@ -211,7 +215,9 @@ protected void shutDown() throws Exception { stopQuietly(dataSubscriberService); stopQuietly(logSubscriberService); logAppender.stop(); - stopQuietly(metricsCollectionService); + if (metricsCollectionServiceStartedByMe) { + stopQuietly(metricsCollectionService); + } previewLevelDBTableService.close(); } @@ -449,7 +455,7 @@ private Principal encryptCredential() { private void stopQuietly(Service service) { try { - service.stopAndWait(); + service.stopAsync().awaitTerminated(); } catch (Exception e) { LOG.warn("Exception when stopping service {}", service, e); } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/DefaultPreviewRunner.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/DefaultPreviewRunner.java index 1754eb19f496..7c2268777a7c 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/DefaultPreviewRunner.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/DefaultPreviewRunner.java @@ -117,6 +117,14 @@ public class DefaultPreviewRunner extends AbstractIdleService implements Preview private final Path previewIdDirPath; private final PreferencesFetcher preferencesFetcher; private final CConfiguration cConf; + private boolean messagingServiceStartedByMe = false; + private boolean dsOpExecServiceStartedByMe = false; + private boolean datasetServiceStartedByMe = false; + private boolean applicationLifecycleServiceStartedByMe = false; + private boolean programRuntimeServiceStartedByMe = false; + private boolean metricsCollectionServiceStartedByMe = false; + private boolean programNotificationSubscriberServiceStartedByMe = false; + private boolean programStopSubscriberServiceStartedByMe = false; @Inject DefaultPreviewRunner(MessagingService messagingService, @@ -329,10 +337,20 @@ protected void startUp() throws Exception { LOG.debug("Starting preview runner service"); StoreDefinition.createAllTables(structuredTableAdmin); if (messagingService instanceof Service) { - ((Service) messagingService).startAndWait(); + Service msgService = (Service) messagingService; + if (msgService.state() == Service.State.NEW) { + msgService.startAsync().awaitRunning(); + messagingServiceStartedByMe = true; + } + } + if (dsOpExecService.state() == Service.State.NEW) { + dsOpExecService.startAsync().awaitRunning(); + dsOpExecServiceStartedByMe = true; + } + if (datasetService.state() == Service.State.NEW) { + datasetService.startAsync().awaitRunning(); + datasetServiceStartedByMe = true; } - dsOpExecService.startAndWait(); - datasetService.startAndWait(); // It is recommended to initialize log appender after datasetService is started, // since log appender instantiates a dataset. @@ -342,13 +360,44 @@ protected void startUp() throws Exception { new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, Constants.Service.PREVIEW_HTTP)); - Futures.allAsList( - applicationLifecycleService.start(), - programRuntimeService.start(), - metricsCollectionService.start(), - programNotificationSubscriberService.start(), - programStopSubscriberService.start() - ).get(); + + if (applicationLifecycleService.state() == Service.State.NEW) { + applicationLifecycleService.startAsync(); + applicationLifecycleServiceStartedByMe = true; + } + if (programRuntimeService.state() == Service.State.NEW) { + programRuntimeService.startAsync(); + programRuntimeServiceStartedByMe = true; + } + if (metricsCollectionService.state() == Service.State.NEW) { + metricsCollectionService.startAsync(); + metricsCollectionServiceStartedByMe = true; + } + if (programNotificationSubscriberService.state() == Service.State.NEW) { + programNotificationSubscriberService.startAsync(); + programNotificationSubscriberServiceStartedByMe = true; + } + if (programStopSubscriberService.state() == Service.State.NEW) { + programStopSubscriberService.startAsync(); + programStopSubscriberServiceStartedByMe = true; + } + + if (applicationLifecycleServiceStartedByMe || applicationLifecycleService.state() == Service.State.STARTING) { + applicationLifecycleService.awaitRunning(); + } + if (programRuntimeServiceStartedByMe || programRuntimeService.state() == Service.State.STARTING) { + programRuntimeService.awaitRunning(); + } + if (metricsCollectionServiceStartedByMe || metricsCollectionService.state() == Service.State.STARTING) { + metricsCollectionService.awaitRunning(); + } + if (programNotificationSubscriberServiceStartedByMe + || programNotificationSubscriberService.state() == Service.State.STARTING) { + programNotificationSubscriberService.awaitRunning(); + } + if (programStopSubscriberServiceStartedByMe || programStopSubscriberService.state() == Service.State.STARTING) { + programStopSubscriberService.awaitRunning(); + } Files.createDirectories(previewIdDirPath); @@ -375,16 +424,30 @@ protected void startUp() throws Exception { @Override protected void shutDown() throws Exception { LOG.debug("Stopping preview runner service"); - programRuntimeService.stopAndWait(); - applicationLifecycleService.stopAndWait(); + if (programRuntimeServiceStartedByMe) { + programRuntimeService.stopAsync().awaitTerminated(); + } + if (applicationLifecycleServiceStartedByMe) { + applicationLifecycleService.stopAsync().awaitTerminated(); + } logAppenderInitializer.close(); - metricsCollectionService.stopAndWait(); - programNotificationSubscriberService.stopAndWait(); - programStopSubscriberService.stopAndWait(); - datasetService.stopAndWait(); - dsOpExecService.stopAndWait(); - if (messagingService instanceof Service) { - ((Service) messagingService).stopAndWait(); + if (metricsCollectionServiceStartedByMe) { + metricsCollectionService.stopAsync().awaitTerminated(); + } + if (programNotificationSubscriberServiceStartedByMe) { + programNotificationSubscriberService.stopAsync().awaitTerminated(); + } + if (programStopSubscriberServiceStartedByMe) { + programStopSubscriberService.stopAsync().awaitTerminated(); + } + if (datasetServiceStartedByMe) { + datasetService.stopAsync().awaitTerminated(); + } + if (dsOpExecServiceStartedByMe) { + dsOpExecService.stopAsync().awaitTerminated(); + } + if (messagingServiceStartedByMe && messagingService instanceof Service) { + ((Service) messagingService).stopAsync().awaitTerminated(); } levelDBTableService.close(); } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/PreviewRunnerTwillRunnable.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/PreviewRunnerTwillRunnable.java index 16eae6e73bfa..19b88766d83a 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/PreviewRunnerTwillRunnable.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/PreviewRunnerTwillRunnable.java @@ -124,7 +124,7 @@ public void initialize(TwillContext context) { @Override public void run() { CompletableFuture future = new CompletableFuture<>(); - previewRunnerManager.addListener(new ServiceListenerAdapter() { + previewRunnerManager.addListener(new Service.Listener() { @Override public void terminated(Service.State from) { future.complete(from); @@ -137,7 +137,7 @@ public void failed(Service.State from, Throwable failure) { }, Threads.SAME_THREAD_EXECUTOR); LOG.debug("Starting preview runner manager"); - previewRunnerManager.start(); + previewRunnerManager.startAsync(); try { Uninterruptibles.getUninterruptibly(future); @@ -150,7 +150,7 @@ public void failed(Service.State from, Throwable failure) { @Override public void stop() { LOG.info("Stopping preview runner manager"); - previewRunnerManager.stop(); + previewRunnerManager.stopAsync(); } @Override diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/AbstractProgramController.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/AbstractProgramController.java index fd3b1ea2cea7..2fa6ae986965 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/AbstractProgramController.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/AbstractProgramController.java @@ -16,9 +16,9 @@ package io.cdap.cdap.internal.app.runtime; -import com.google.common.base.Objects; import com.google.common.base.Preconditions; import com.google.common.base.Throwables; +import java.util.Objects; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; @@ -471,7 +471,7 @@ public boolean equals(Object o) { // Only compare with the listener ListenerCaller other = (ListenerCaller) o; - return Objects.equal(listener, other.listener); + return Objects.equals(listener, other.listener); } @Override diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/AbstractProgramRunnerWithPlugin.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/AbstractProgramRunnerWithPlugin.java index 4681ade570e4..79077453c95a 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/AbstractProgramRunnerWithPlugin.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/AbstractProgramRunnerWithPlugin.java @@ -61,7 +61,7 @@ protected PluginInstantiator createPluginInstantiator(ProgramOptions options, * Creates a service listener to cleanup closeables. */ protected Service.Listener createRuntimeServiceListener(final Iterable closeables) { - return new ServiceListenerAdapter() { + return new Service.Listener() { @Override public void terminated(Service.State from) { closeAllQuietly(closeables); @@ -76,7 +76,13 @@ public void failed(Service.State from, @Nullable final Throwable failure) { protected void closeAllQuietly(Iterable closeables) { for (Closeable c : closeables) { - Closeables.closeQuietly(c); + try { + if (c != null) { + c.close(); + } + } catch (java.io.IOException e) { + // Ignore + } } } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/LocalizationUtils.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/LocalizationUtils.java index 7c8bab098fdb..bddb258306e3 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/LocalizationUtils.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/LocalizationUtils.java @@ -84,7 +84,7 @@ private static File getFileToLocalize(LocalizeResource resource, File tempDir) URL url = uri.toURL(); String name = new File(uri.getPath()).getName(); File tempFile = new File(tempDir, name); - Files.copy(Resources.newInputStreamSupplier(url), tempFile); + Resources.asByteSource(url).copyTo(Files.asByteSink(tempFile)); return tempFile; } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/ProgramControllerServiceAdapter.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/ProgramControllerServiceAdapter.java index 5a1c4b8159a0..588ece9c8384 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/ProgramControllerServiceAdapter.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/ProgramControllerServiceAdapter.java @@ -64,7 +64,7 @@ protected void doStop() throws Exception { stopRequested = true; long gracefulTimeoutMillis = getGracefulTimeoutMillis(); if (gracefulTimeoutMillis < 0) { - service.stopAndWait(); + service.stopAsync().awaitTerminated(); } else { gracefulStop(gracefulTimeoutMillis); } @@ -77,7 +77,7 @@ protected void doStop() throws Exception { * supports graceful termination with timeout. */ protected void gracefulStop(long gracefulTimeoutMillis) { - service.stopAndWait(); + service.stopAsync().awaitTerminated(); } @Override @@ -86,7 +86,7 @@ protected void doCommand(String name, Object value) throws Exception { } private void listenToRuntimeState(Service service) { - service.addListener(new ServiceListenerAdapter() { + service.addListener(new Service.Listener() { @Override public void running() { started(); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/ProgramRunners.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/ProgramRunners.java index c5b402e31a6c..3869296a2b85 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/ProgramRunners.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/ProgramRunners.java @@ -78,7 +78,20 @@ public static void startAsUser(String user, final Service service) runAsUser(user, new Callable>() { @Override public ListenableFuture call() throws Exception { - return service.start(); + com.google.common.util.concurrent.SettableFuture future = + com.google.common.util.concurrent.SettableFuture.create(); + service.addListener(new Service.Listener() { + @Override + public void running() { + future.set(Service.State.RUNNING); + } + @Override + public void failed(Service.State from, Throwable failure) { + future.setException(failure); + } + }, com.google.common.util.concurrent.MoreExecutors.directExecutor()); + service.startAsync(); + return future; } }); } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/artifact/AbstractArtifactManager.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/artifact/AbstractArtifactManager.java index 4bc0ff98aa64..871a4c1c3f28 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/artifact/AbstractArtifactManager.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/artifact/AbstractArtifactManager.java @@ -113,8 +113,16 @@ private ClassLoaderCleanup(DirectoryClassLoader directoryClassLoader, @Override public void close() throws IOException { - Closeables.closeQuietly(directoryClassLoader); - Closeables.closeQuietly(folder); + try { + directoryClassLoader.close(); + } catch (IOException e) { + // Ignore + } + try { + folder.close(); + } catch (IOException e) { + // Ignore + } } } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/artifact/ArtifactClassLoaderFactory.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/artifact/ArtifactClassLoaderFactory.java index 2704901782c1..9cd8a58fcad4 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/artifact/ArtifactClassLoaderFactory.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/artifact/ArtifactClassLoaderFactory.java @@ -33,6 +33,7 @@ import io.cdap.cdap.security.impersonation.EntityImpersonator; import java.io.Closeable; import java.io.File; +import java.io.IOException; import java.util.Iterator; import javax.annotation.Nullable; import org.apache.twill.filesystem.Location; @@ -104,10 +105,18 @@ CloseableClassLoader createClassLoader(File unpackDir) { final ClassLoader finalSparkClassLoader = sparkClassLoader; return new CloseableClassLoader(programClassLoader, () -> { if (finalProgramClassLoader instanceof Closeable) { - Closeables.closeQuietly((Closeable) finalProgramClassLoader); + try { + ((Closeable) finalProgramClassLoader).close(); + } catch (IOException e) { + // Ignore + } } if (finalSparkClassLoader instanceof Closeable) { - Closeables.closeQuietly((Closeable) finalSparkClassLoader); + try { + ((Closeable) finalSparkClassLoader).close(); + } catch (IOException e) { + // Ignore + } } }); } @@ -130,8 +139,16 @@ CloseableClassLoader createClassLoader(Location artifactLocation, CloseableClassLoader classLoader = createClassLoader(classLoaderFolder.getDir()); return new CloseableClassLoader(classLoader, () -> { - Closeables.closeQuietly(classLoader); - Closeables.closeQuietly(classLoaderFolder); + try { + classLoader.close(); + } catch (IOException e) { + // Ignore + } + try { + classLoaderFolder.close(); + } catch (IOException e) { + // Ignore + } }); } catch (Exception e) { throw Throwables.propagate(e); @@ -167,8 +184,16 @@ CloseableClassLoader createClassLoader(Iterator artifactLocations, entityImpersonator); return new CloseableClassLoader(new DirectoryClassLoader(classLoaderFolder.getDir(), parentClassLoader, "lib"), () -> { - Closeables.closeQuietly(parentClassLoader); - Closeables.closeQuietly(classLoaderFolder); + try { + parentClassLoader.close(); + } catch (IOException e) { + // Ignore + } + try { + classLoaderFolder.close(); + } catch (IOException e) { + // Ignore + } }); } catch (Exception e) { throw Throwables.propagate(e); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/artifact/DefaultArtifactInspector.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/artifact/DefaultArtifactInspector.java index 662695ff324d..fdac75c5b7e0 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/artifact/DefaultArtifactInspector.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/artifact/DefaultArtifactInspector.java @@ -624,7 +624,7 @@ private boolean isPlugin(String className, ClassLoader classLoader) { // Use ASM to inspect the class bytecode to see if it is annotated with @Plugin final boolean[] isPlugin = new boolean[1]; ClassReader cr = new ClassReader(is); - cr.accept(new ClassVisitor(Opcodes.ASM7) { + cr.accept(new ClassVisitor(Opcodes.ASM9) { @Override public AnnotationVisitor visitAnnotation(String desc, boolean visible) { if (Plugin.class.getName().equals(Type.getType(desc).getClassName()) && visible) { diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceClassLoader.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceClassLoader.java index 2d3c264cda65..c022ee45ba3d 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceClassLoader.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceClassLoader.java @@ -153,7 +153,11 @@ public MapReduceTaskContextProvider getTaskContextProvider() { taskContextProvider = Optional.fromNullable(taskContextProvider) .or(taskContextProviderSupplier); } - taskContextProvider.startAndWait(); + if (taskContextProvider.state() == Service.State.NEW) { + taskContextProvider.startAsync().awaitRunning(); + } else { + taskContextProvider.awaitRunning(); + } return taskContextProvider; } @@ -196,7 +200,7 @@ public void close() { if (provider != null) { Service.State state = provider.state(); if (state == Service.State.STARTING || state == Service.State.RUNNING) { - provider.stopAndWait(); + provider.stopAsync().awaitTerminated(); } } } catch (Exception e) { diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceProgramRunner.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceProgramRunner.java index d1a39f718efc..724003ce5d8e 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceProgramRunner.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceProgramRunner.java @@ -231,7 +231,7 @@ public ProgramController run(final Program program, ProgramOptions options) { // tries to access cdap data. For example, writing to a FileSet will fail, as the yarn user will // be running the job, but the data directory will be owned by cdap. if (MapReduceTaskContextProvider.isLocal(hConf) || UserGroupInformation.isSecurityEnabled()) { - mapReduceRuntimeService.start(); + mapReduceRuntimeService.startAsync(); } else { ProgramRunners.startAsUser(cConf.get(Constants.CFG_HDFS_USER), mapReduceRuntimeService); } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceRuntimeService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceRuntimeService.java index c858bbf08fb4..31c9256351f4 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceRuntimeService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceRuntimeService.java @@ -190,7 +190,7 @@ public void destroy() { } @Override - protected String getServiceName() { + protected String serviceName() { return "MapReduceRunner-" + specification.getName(); } @@ -454,7 +454,7 @@ public void run() { } }); t.setDaemon(true); - t.setName(getServiceName()); + t.setName(serviceName()); t.start(); } }; @@ -1012,7 +1012,10 @@ private Location createPluginArchive(Location targetDir) throws IOException { */ private Location copyFileToLocation(File file, Location targetDir) throws IOException { Location targetLocation = targetDir.append(file.getName()).getTempFile(".jar"); - Files.copy(file, Locations.newOutputSupplier(targetLocation)); + try (java.io.InputStream in = new java.io.FileInputStream(file); + java.io.OutputStream out = targetLocation.getOutputStream()) { + com.google.common.io.ByteStreams.copy(in, out); + } return targetLocation; } @@ -1024,8 +1027,10 @@ private Location copyFileToLocation(File file, Location targetDir) throws IOExce private Location copyProgramJar(Location targetDir) throws IOException { Location programJarCopy = targetDir.append("program.jar"); - ByteStreams.copy(Locations.newInputSupplier(programJarLocation), - Locations.newOutputSupplier(programJarCopy)); + try (java.io.InputStream in = programJarLocation.getInputStream(); + java.io.OutputStream out = programJarCopy.getOutputStream()) { + com.google.common.io.ByteStreams.copy(in, out); + } LOG.debug("Copied Program Jar to {}, source: {}", programJarCopy, programJarLocation); return programJarCopy; } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/distributed/DistributedMapReduceTaskContextProvider.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/distributed/DistributedMapReduceTaskContextProvider.java index fcc1cd564e9c..90804b29c31a 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/distributed/DistributedMapReduceTaskContextProvider.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/distributed/DistributedMapReduceTaskContextProvider.java @@ -91,7 +91,7 @@ protected void startUp() throws Exception { } for (Service service : coreServices) { - service.startAndWait(); + service.startAsync().awaitRunning(); } } catch (Exception e) { // Try our best to stop services. Chain stop guarantees it will stop everything, even some of them failed. @@ -107,12 +107,12 @@ protected void startUp() throws Exception { @Override protected void shutDown() throws Exception { super.shutDown(); - Closeables.closeQuietly(logAppenderInitializer); + logAppenderInitializer.close(); Exception failure = null; for (Service service : (Iterable) coreServices::descendingIterator) { try { - service.stopAndWait(); + service.stopAsync().awaitTerminated(); } catch (Exception e) { if (failure != null) { failure.addSuppressed(e); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/AbstractProgramTwillRunnable.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/AbstractProgramTwillRunnable.java index 6c48872f7224..20be4f43bde0 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/AbstractProgramTwillRunnable.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/AbstractProgramTwillRunnable.java @@ -96,7 +96,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.bridge.SLF4JBridgeHandler; -import sun.net.InetAddressCachePolicy; /** * A {@link TwillRunnable} for running a program through a {@link ProgramRunner}. @@ -173,15 +172,7 @@ private void doInitialize(File programOptionFile) throws Exception { controllerFuture = new CompletableFuture<>(); programCompletion = new CompletableFuture<>(); - // Make sure InetAddressCachePolicy is not set to FOREVER. - // This is needed because if InetAddressCachePolicy is loaded after System#setSecurityManager() call, - // caching policy is set to FOREVER. With FOREVER, caching policy, this twill runnable will not be able to - // reach out to other services within the cluster as IP is cached forever. - // Look at CDAP-20781 for more information. - if (InetAddressCachePolicy.get() == InetAddressCachePolicy.FOREVER) { - LOG.warn("InetAddress cache policy is set to FOREVER. This will ips to be cached forever. " - + "Ensure cache policy is not set to FOREVER."); - } + // Setup process wide settings Thread.setDefaultUncaughtExceptionHandler(new UncaughtExceptionHandler()); @@ -296,9 +287,19 @@ public void error(Throwable cause) { LOG.info("Program run {} completed. Releasing resources.", programRunId); // Close the Program and the ProgramRunner - Closeables.closeQuietly(program); + try { + if (program != null) { + program.close(); + } + } catch (Exception e) { + // Ignore + } if (programRunner instanceof Closeable) { - Closeables.closeQuietly((Closeable) programRunner); + try { + ((Closeable) programRunner).close(); + } catch (java.io.IOException e) { + // Ignore + } } stopCoreServices(); @@ -483,16 +484,22 @@ private void addOnPremiseServices(Injector injector, ProgramOptions programOptio private void startCoreServices() { // Starts the core services for (Service service : coreServices) { - service.startAndWait(); + service.startAsync().awaitRunning(); } } private void stopCoreServices() { - Closeables.closeQuietly(logAppenderInitializer); + try { + if (logAppenderInitializer != null) { + logAppenderInitializer.close(); + } + } catch (Exception e) { + // Ignore + } // Stop all services. Reverse the order. for (Service service : (Iterable) coreServices::descendingIterator) { try { - service.stopAndWait(); + service.stopAsync().awaitTerminated(); } catch (Exception e) { LOG.warn("Exception raised when stopping service {} during program termination.", service, e); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/DistributedWorkflowProgramRunner.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/DistributedWorkflowProgramRunner.java index b3822eb518c5..e1a3c56ab5d5 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/DistributedWorkflowProgramRunner.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/DistributedWorkflowProgramRunner.java @@ -169,7 +169,11 @@ protected void setupLaunchConfig(ProgramLaunchConfig launchConfig, Program progr } } finally { if (runner instanceof Closeable) { - Closeables.closeQuietly((Closeable) runner); + try { + ((Closeable) runner).close(); + } catch (IOException e) { + // Ignore + } } } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/LocalizeResource.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/LocalizeResource.java index fdc664bb3b36..52d419a00413 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/LocalizeResource.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/LocalizeResource.java @@ -16,7 +16,7 @@ package io.cdap.cdap.internal.app.runtime.distributed; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import java.io.File; import java.net.URI; @@ -51,7 +51,7 @@ public URI getURI() { @Override public String toString() { - return Objects.toStringHelper(this) + return com.google.common.base.MoreObjects.toStringHelper(this) .add("archive", archive) .add("uri", uri) .toString(); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/AbstractRuntimeTwillPreparer.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/AbstractRuntimeTwillPreparer.java index 9a8001dfabfd..b6f2ac22a80c 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/AbstractRuntimeTwillPreparer.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/AbstractRuntimeTwillPreparer.java @@ -520,10 +520,10 @@ private void createApplicationJar(ApplicationBundler bundler, .collect(Collectors.toList()); Hasher hasher = Hashing.md5().newHasher(); for (String name : classList) { - hasher.putString(name); + hasher.putString(name, java.nio.charset.StandardCharsets.UTF_8); } // Add cdap version to the hash so that application jars are distinguishable when upgrade happens. - hasher.putString(ProjectInfo.getVersion().toString()); + hasher.putString(ProjectInfo.getVersion().toString(), java.nio.charset.StandardCharsets.UTF_8); // Only depends on class list and cdap version so that it can be reused across different launches String hashVal = hasher.hash().toString(); String name = hashVal + "-" + Constants.Files.APPLICATION_JAR; @@ -674,8 +674,9 @@ private void saveClassPaths(Path targetDir) throws IOException { } private void saveArguments(Arguments arguments, final Path targetPath) throws IOException { - ArgumentsCodec.encode(arguments, - () -> Files.newBufferedWriter(targetPath, StandardCharsets.UTF_8)); + try (Writer writer = Files.newBufferedWriter(targetPath, StandardCharsets.UTF_8)) { + ArgumentsCodec.encode(arguments, writer); + } } /** diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionJobMain.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionJobMain.java index c60dbff4d03a..a2144f6e6b42 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionJobMain.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionJobMain.java @@ -127,7 +127,7 @@ private void doMain(String[] args) throws Exception { @VisibleForTesting RemoteExecutionRuntimeJobEnvironment initialize(CConfiguration cConf) throws Exception { zkServer = InMemoryZKServer.builder().build(); - zkServer.startAndWait(); + zkServer.startAsync().awaitRunning(); InetSocketAddress zkAddr = ResolvingDiscoverable.resolve(zkServer.getLocalAddress()); String zkConnectStr = String.format("%s:%d", zkAddr.getHostString(), zkAddr.getPort()); @@ -214,7 +214,7 @@ void destroy() { if (zkServer != null) { try { - zkServer.stopAndWait(); + zkServer.stopAsync().awaitTerminated(); } catch (Exception e) { LOG.warn("Failed to stop ZK server", e); } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionService.java index 1075b5925465..a762bce005a7 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionService.java @@ -88,7 +88,7 @@ protected final long runTask() throws Exception { LOG.debug("Program {} is not running", programRunId); programStateWriter.error(programRunId, new IllegalStateException("Program terminated " + programRunId)); - stop(); + stopAsync(); return 0; } nextCheckRunningMillis = now + pollTimeMillis * 10; diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionTwillController.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionTwillController.java index 906197359e01..14c5cbf97b84 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionTwillController.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionTwillController.java @@ -88,7 +88,7 @@ class RemoteExecutionTwillController implements TwillController { completion.completeExceptionally(throwable); return RemoteExecutionTwillController.this; }); - service.addListener(new ServiceListenerAdapter() { + service.addListener(new Service.Listener() { @Override public void terminated(Service.State from) { if (terminateOnServiceStop) { @@ -113,12 +113,12 @@ public void failed(Service.State from, Throwable failure) { public void release() { terminateOnServiceStop = false; - executionService.stop(); + executionService.stopAsync(); } public void complete() { terminateOnServiceStop = true; - executionService.stop(); + executionService.stopAsync(); try { RuntimeJobStatus status; RetryStrategy retryStrategy = RetryStrategies.timeLimit( diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionTwillPreparer.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionTwillPreparer.java index 73612175c011..81e1e4d76473 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionTwillPreparer.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionTwillPreparer.java @@ -250,7 +250,8 @@ private void localizeFiles( String localizedFile = localizedFiles.get(uri); if (localizedFile == null) { String fileName = - Hashing.md5().hashString(uri.toString()).toString() + "-" + getFileName(uri); + Hashing.md5().hashString(uri.toString(), + java.nio.charset.StandardCharsets.UTF_8).toString() + "-" + getFileName(uri); localizedFile = localizedDir + "/" + fileName; try (InputStream inputStream = openUri(uri)) { LOG.debug( diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionTwillRunnerService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionTwillRunnerService.java index 97b8b9ff0829..b0b1a01bb9fc 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionTwillRunnerService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionTwillRunnerService.java @@ -217,7 +217,7 @@ public void stop() { try { if (EnumSet.of(Service.State.STARTING, Service.State.RUNNING) .contains(serviceSocksProxy.state())) { - serviceSocksProxy.stopAndWait(); + serviceSocksProxy.stopAsync().awaitTerminated(); } } catch (Exception e) { LOG.warn("Exception raised when stopping runtime monitor socks proxy", e); @@ -389,7 +389,7 @@ private TwillPreparer createPreparer(CConfiguration cConf, Configuration hConf, private int getServiceSocksProxyPort() { if (serviceSocksProxy.state() == Service.State.NEW) { // It's ok to have multiple threads calling start if the proxy is not running. - serviceSocksProxy.startAndWait(); + serviceSocksProxy.startAsync().awaitRunning(); } return serviceSocksProxy.getBindAddress().getPort(); } @@ -428,7 +428,7 @@ private Location generateAndSaveServiceProxySecret(ProgramRunId programRunId, Lo String secret = Hashing.sha1().newHasher() .putBytes(salt) - .putString(programRunId.getRun()) + .putString(programRunId.getRun(), StandardCharsets.UTF_8) .hash().toString(); Location location = keysDir.append(Constants.RuntimeMonitor.SERVICE_PROXY_PASSWORD_FILE); @@ -549,7 +549,7 @@ protected void monitorController(ProgramRunId programRunId, CompletableFuture startupTaskCompletion, RemoteExecutionTwillController controller, RemoteExecutionService remoteExecutionService) { - startupTaskCompletion.thenAccept(o -> remoteExecutionService.start()); + startupTaskCompletion.thenAccept(o -> remoteExecutionService.startAsync()); // On this controller termination, make sure it is removed from the controllers map and have resources released. controller.onTerminated(() -> { @@ -751,7 +751,7 @@ private RemoteExecutionService createRemoteExecutionService(ProgramRunId program programStateWriter, scheduler); LOG.debug("Monitor program run {} with SSH config {}", programRunId, sshConfig); String proxySecret = clusterKeyInfo.getServerProxySecret(); - remoteExecutionService.addListener(new ServiceListenerAdapter() { + remoteExecutionService.addListener(new Service.Listener() { @Override public void running() { serviceSocksProxyAuthenticator.add(proxySecret); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/SSHRemoteExecutionService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/SSHRemoteExecutionService.java index f3249c088e77..46979d77f0f3 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/SSHRemoteExecutionService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/SSHRemoteExecutionService.java @@ -61,13 +61,25 @@ protected void doRunTask() throws Exception { if (sshSession != null && sshSession.isAlive()) { return; } - Closeables.closeQuietly(sshSession); + try { + if (sshSession != null) { + sshSession.close(); + } + } catch (Exception e) { + // Ignore + } sshSession = createServiceProxyTunnel(); } @Override protected void doShutdown() { - Closeables.closeQuietly(sshSession); + try { + if (sshSession != null) { + sshSession.close(); + } + } catch (Exception e) { + // Ignore + } LOG.debug("Stopped ssh service for run {}", getProgramRunId()); } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/runtimejob/DefaultRuntimeJob.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/runtimejob/DefaultRuntimeJob.java index 94d1d33eba49..33df437a6730 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/runtimejob/DefaultRuntimeJob.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/runtimejob/DefaultRuntimeJob.java @@ -291,7 +291,11 @@ public void run(RuntimeJobEnvironment runtimeJobEnv) throws Exception { programCompletion.get(); } finally { if (programRunner instanceof Closeable) { - Closeables.closeQuietly((Closeable) programRunner); + try { + ((Closeable) programRunner).close(); + } catch (IOException e) { + // Ignore + } } } } catch (Throwable t) { @@ -527,7 +531,7 @@ private void startCoreServices(Deque coreServices) { // Starts the core services for (Service service : coreServices) { LOG.debug("Starting core service {}", service); - service.startAndWait(); + service.startAsync().awaitRunning(); } } @@ -540,7 +544,7 @@ private void stopCoreServices(Deque coreServices, for (Service service : (Iterable) coreServices::descendingIterator) { LOG.debug("Stopping core service {}", service); try { - service.stopAndWait(); + service.stopAsync().awaitTerminated(); } catch (Exception e) { LOG.warn( "Exception raised when stopping service {} during program termination.", @@ -624,7 +628,7 @@ private void monitorServicesHealth(ProgramRunId programRunId, ProgramController controller) { for (Service service : services) { - service.addListener(new ServiceListenerAdapter() { + service.addListener(new Service.Listener() { @Override public void failed(Service.State from, Throwable failure) { LOG.error( @@ -721,7 +725,7 @@ private static final class TrafficRelayService extends AbstractIdleService { protected void startUp() throws Exception { // Bind the traffic relay on the host, not on the loopback interface. It needs to be accessible from all workers. relayServer = new TrafficRelayServer(InetAddress.getLocalHost(), this::getTrafficRelayTarget); - relayServer.startAndWait(); + relayServer.startAsync().awaitRunning(); // Set the traffic relay service address to cConf. It will be used as the proxy address for all worker processes Networks.setAddress(cConf, Constants.RuntimeMonitor.SERVICE_PROXY_ADDRESS, @@ -732,7 +736,7 @@ protected void startUp() throws Exception { @Override protected void shutDown() { - relayServer.stopAndWait(); + relayServer.stopAsync().awaitTerminated(); getServiceProxyFile().delete(); } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/monitor/AbstractServiceRoutingHandler.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/monitor/AbstractServiceRoutingHandler.java index 3b49421b5dd5..b6a618d791b3 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/monitor/AbstractServiceRoutingHandler.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/monitor/AbstractServiceRoutingHandler.java @@ -305,13 +305,13 @@ public ByteBuf nextChunk() throws Exception { @Override public void finished() { - Closeables.closeQuietly(responseInfo); + responseInfo.close(); } @Override public void handleError(@Nullable Throwable cause) { LOG.warn("Exception raised when handling request to {}", url, cause); - Closeables.closeQuietly(responseInfo); + responseInfo.close(); } } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/monitor/DirectRuntimeRequestValidator.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/monitor/DirectRuntimeRequestValidator.java index 0c90be3fcaa5..54ccf314b91f 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/monitor/DirectRuntimeRequestValidator.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/monitor/DirectRuntimeRequestValidator.java @@ -16,7 +16,7 @@ package io.cdap.cdap.internal.app.runtime.monitor; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; @@ -165,24 +165,24 @@ private void insertRunRecord(ProgramRunId programRunId, RunRecordDetail runRecor store.recordProgramStart(programRunId, null, runRecord.getSystemArgs(), runRecord.getSourceId()); store.recordProgramRunning(programRunId, - Objects.firstNonNull(runRecord.getRunTs(), System.currentTimeMillis()), + MoreObjects.firstNonNull(runRecord.getRunTs(), System.currentTimeMillis()), null, runRecord.getSourceId()); switch (runRecord.getStatus()) { case SUSPENDED: store.recordProgramSuspend(programRunId, runRecord.getSourceId(), - Objects.firstNonNull(runRecord.getSuspendTs(), System.currentTimeMillis())); + MoreObjects.firstNonNull(runRecord.getSuspendTs(), System.currentTimeMillis())); break; case STOPPING: store.recordProgramStopping(programRunId, runRecord.getSourceId(), - Objects.firstNonNull(runRecord.getStoppingTs(), System.currentTimeMillis()), + MoreObjects.firstNonNull(runRecord.getStoppingTs(), System.currentTimeMillis()), // if terminate timestamp is null we will shut down gracefully - Objects.firstNonNull(runRecord.getTerminateTs(), Long.MAX_VALUE)); + MoreObjects.firstNonNull(runRecord.getTerminateTs(), Long.MAX_VALUE)); break; case COMPLETED: case KILLED: case FAILED: store.recordProgramStop(programRunId, - Objects.firstNonNull(runRecord.getStopTs(), System.currentTimeMillis()), + MoreObjects.firstNonNull(runRecord.getStopTs(), System.currentTimeMillis()), runRecord.getStatus(), null, runRecord.getSourceId()); // We don't need to retain records for terminated programs, hence just delete it store.deleteRunIfTerminated(programRunId, runRecord.getSourceId()); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/monitor/RuntimeClientService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/monitor/RuntimeClientService.java index 5e4bd6e6f1ed..fac00e8d7827 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/monitor/RuntimeClientService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/monitor/RuntimeClientService.java @@ -141,7 +141,7 @@ > getProgramCompletionDetails().getEndTimestamp())) { LOG.debug( "Program {} terminated. Shutting down runtime client service.", programRunId); - stop(); + stopAsync(); } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/monitor/RuntimeHandler.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/monitor/RuntimeHandler.java index 75f040e91524..d3a6950f824e 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/monitor/RuntimeHandler.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/monitor/RuntimeHandler.java @@ -227,7 +227,11 @@ public void finished(HttpResponder responder) { @Override public void handleError(Throwable cause) { LOG.error("Failed to write spark event logs for {}", programRunId, cause); - Closeables.closeQuietly(os); + try { + os.close(); + } catch (IOException e) { + // Ignore + } try { location.delete(); } catch (IOException e) { diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/plugin/PluginInstantiator.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/plugin/PluginInstantiator.java index 60a50516435c..9e06cacb5aca 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/plugin/PluginInstantiator.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/plugin/PluginInstantiator.java @@ -517,7 +517,13 @@ public void close() throws IOException { // Cleanup the ClassLoader cache and the temporary directory for the expanded plugin jar. classLoaders.invalidateAll(); if (ownedParentClassLoader) { - Closeables.closeQuietly((Closeable) parentClassLoader); + try { + if (parentClassLoader instanceof Closeable) { + ((Closeable) parentClassLoader).close(); + } + } catch (IOException e) { + // Ignore + } } try { DirUtils.deleteDirectoryContents(tmpDir); @@ -618,7 +624,13 @@ private static final class ClassLoaderRemovalListener implements @Override public void onRemoval(RemovalNotification notification) { - Closeables.closeQuietly(notification.getValue()); + try { + if (notification.getValue() != null) { + notification.getValue().close(); + } + } catch (IOException e) { + // Ignore + } } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/DistributedTimeSchedulerService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/DistributedTimeSchedulerService.java index f91b134705ab..f00a2837c6d4 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/DistributedTimeSchedulerService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/DistributedTimeSchedulerService.java @@ -73,13 +73,13 @@ protected void doStop() { protected void startUp() throws Exception { LOG.info("Starting scheduler."); // RetryOnStartFailureservice#startAndWait returns before its service's startAndWait completes - serviceDelegate.startAndWait(); + serviceDelegate.startAsync().awaitRunning(); startUpLatch.await(); } @Override protected void shutDown() throws Exception { LOG.info("Stopping scheduler."); - serviceDelegate.stopAndWait(); + serviceDelegate.stopAsync().awaitTerminated(); } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/queue/JobKey.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/queue/JobKey.java index 7267b32df0d6..ba9f6e973cda 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/queue/JobKey.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/queue/JobKey.java @@ -16,8 +16,9 @@ package io.cdap.cdap.internal.app.runtime.schedule.queue; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import io.cdap.cdap.proto.id.ScheduleId; +import java.util.Objects; /** * Uniquely identifies a Job. @@ -57,18 +58,18 @@ public boolean equals(Object o) { JobKey that = (JobKey) o; - return Objects.equal(this.scheduleId, that.scheduleId) - && Objects.equal(this.generationId, that.generationId); + return Objects.equals(this.scheduleId, that.scheduleId) + && Objects.equals(this.generationId, that.generationId); } @Override public int hashCode() { - return Objects.hashCode(scheduleId, generationId); + return Objects.hash(scheduleId, generationId); } @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("scheduleId", scheduleId) .add("generationId", generationId) .toString(); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/queue/JobQueueTable.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/queue/JobQueueTable.java index 19cc03e7669d..66c29165ac9b 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/queue/JobQueueTable.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/queue/JobQueueTable.java @@ -425,9 +425,9 @@ int getPartition(ScheduleId scheduleId) { // Similar to ScheduleId#hashCode, but that is not consistent across runtimes due to how Enum#hashCode works. // Ensure that the hash won't change across runtimes: int hash = Hashing.murmur3_32().newHasher() - .putString(scheduleId.getNamespace()) - .putString(scheduleId.getApplication()) - .putString(scheduleId.getSchedule()) + .putString(scheduleId.getNamespace(), java.nio.charset.StandardCharsets.UTF_8) + .putString(scheduleId.getApplication(), java.nio.charset.StandardCharsets.UTF_8) + .putString(scheduleId.getSchedule(), java.nio.charset.StandardCharsets.UTF_8) .hash().asInt(); return Math.abs(hash) % numPartitions; } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/queue/SimpleJob.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/queue/SimpleJob.java index 37b78b0e10cd..15665babc376 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/queue/SimpleJob.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/queue/SimpleJob.java @@ -16,8 +16,9 @@ package io.cdap.cdap.internal.app.runtime.schedule.queue; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; +import java.util.Objects; import io.cdap.cdap.internal.app.runtime.schedule.ProgramSchedule; import io.cdap.cdap.proto.Notification; import java.util.List; @@ -97,25 +98,25 @@ public boolean equals(Object o) { SimpleJob that = (SimpleJob) o; - return Objects.equal(this.schedule, that.schedule) - && Objects.equal(this.creationTime, that.creationTime) - && Objects.equal(this.jobKey, that.jobKey) - && Objects.equal(this.notifications, that.notifications) - && Objects.equal(this.state, that.state) - && Objects.equal(this.scheduleLastUpdatedTime, that.scheduleLastUpdatedTime) - && Objects.equal(this.deleteTimeMillis, that.deleteTimeMillis); + return Objects.equals(this.schedule, that.schedule) + && Objects.equals(this.creationTime, that.creationTime) + && Objects.equals(this.jobKey, that.jobKey) + && Objects.equals(this.notifications, that.notifications) + && Objects.equals(this.state, that.state) + && Objects.equals(this.scheduleLastUpdatedTime, that.scheduleLastUpdatedTime) + && Objects.equals(this.deleteTimeMillis, that.deleteTimeMillis); } @Override public int hashCode() { - return Objects.hashCode(schedule, creationTime, jobKey, notifications, state, + return Objects.hash(schedule, creationTime, jobKey, notifications, state, scheduleLastUpdatedTime, deleteTimeMillis); } @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("schedule", schedule) .add("creationTime", creationTime) .add("jobKey", jobKey) diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/service/ServiceProgramRunner.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/service/ServiceProgramRunner.java index 5f8dbf0a7ffb..32dd6062e136 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/service/ServiceProgramRunner.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/service/ServiceProgramRunner.java @@ -195,10 +195,14 @@ public ProgramController run(Program program, ProgramOptions options) { ProgramController controller = new ServiceProgramControllerAdapter(component, program.getId().run(runId)); - component.start(); + component.startAsync(); return controller; } catch (Throwable t) { - Closeables.closeQuietly(pluginInstantiator); + try { + pluginInstantiator.close(); + } catch (java.io.IOException e) { + // Ignore to avoid masking original exception + } throw t; } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/service/http/HttpHandlerFactory.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/service/http/HttpHandlerFactory.java index 359aacc8d691..22b5b16d4589 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/service/http/HttpHandlerFactory.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/service/http/HttpHandlerFactory.java @@ -111,7 +111,7 @@ public void validateHttpHandler(Iterable handlers) { createHttpHandler(type, new VerificationDelegateContext<>(handler), metricsContext)); } catch (Exception e) { throw new IllegalArgumentException( - "Invalid http handler class " + handler.getClass().getName()); + "Invalid http handler class " + handler.getClass().getName(), e); } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/service/http/HttpHandlerGenerator.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/service/http/HttpHandlerGenerator.java index 614e2c29faf9..057631a90a12 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/service/http/HttpHandlerGenerator.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/service/http/HttpHandlerGenerator.java @@ -150,7 +150,7 @@ ClassDefinition generate(TypeToken delegateType, String pathPrefix) throws IO ClassWriter classWriter = new ClassWriter(ClassWriter.COMPUTE_FRAMES); String internalName = Type.getInternalName(rawType); - String className = internalName + Hashing.md5().hashString(internalName); + String className = internalName + Hashing.md5().hashString(internalName, java.nio.charset.StandardCharsets.UTF_8); // Generate the class Type classType = Type.getObjectType(className); @@ -196,7 +196,7 @@ private void inspectHandler(final TypeToken delegateType, final TypeToken .getResourceAsStream(Type.getInternalName(rawType) + ".class") ) { ClassReader classReader = new ClassReader(sourceBytes); - classReader.accept(new ClassVisitor(Opcodes.ASM5) { + classReader.accept(new ClassVisitor(Opcodes.ASM9) { // Only need to visit @Path at the class level if we are inspecting the user handler class private final boolean inspectDelegate = delegateType.equals(inspectType); @@ -215,7 +215,7 @@ public AnnotationVisitor visitAnnotation(String desc, boolean visible) { if (inspectDelegate && type.equals(Type.getType(Path.class))) { visitedPath = true; AnnotationVisitor annotationVisitor = classWriter.visitAnnotation(desc, visible); - return new AnnotationVisitor(Opcodes.ASM5, annotationVisitor) { + return new AnnotationVisitor(Opcodes.ASM9, annotationVisitor) { @Override public void visit(String name, Object value) { // "value" is the key for the Path annotation string. @@ -374,7 +374,7 @@ private class HandlerMethodVisitor extends MethodVisitor { HandlerMethodVisitor(TypeToken delegateType, MethodVisitor mv, String desc, String signature, int access, String name, String[] exceptions, Type classType, ClassWriter classWriter, List> preservedClasses) { - super(Opcodes.ASM5, mv); + super(Opcodes.ASM9, mv); this.delegateType = delegateType; this.desc = desc; this.signature = signature; @@ -392,7 +392,7 @@ private class HandlerMethodVisitor extends MethodVisitor { public AnnotationVisitor visitAnnotation(String desc, boolean visible) { // Memorize all visible annotations if (visible) { - AnnotationNode annotationNode = new AnnotationNode(Opcodes.ASM5, desc); + AnnotationNode annotationNode = new AnnotationNode(Opcodes.ASM9, desc); annotations.add(annotationNode); return annotationNode; } @@ -404,7 +404,7 @@ public AnnotationVisitor visitParameterAnnotation(int parameter, String desc, bo // Memorize all visible annotations for each parameter. // It needs to store in a Multimap because there can be multiple annotations per parameter. if (visible) { - AnnotationNode annotationNode = new AnnotationNode(Opcodes.ASM5, desc); + AnnotationNode annotationNode = new AnnotationNode(Opcodes.ASM9, desc); paramAnnotations.put(parameter, annotationNode); return annotationNode; } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/service/http/LocationHttpContentProducer.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/service/http/LocationHttpContentProducer.java index 2c6f68cc12bb..fdb4ebd740f9 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/service/http/LocationHttpContentProducer.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/service/http/LocationHttpContentProducer.java @@ -82,7 +82,11 @@ public void onFinish() throws Exception { @Override @TransactionPolicy(TransactionControl.EXPLICIT) public void onError(Throwable failureCause) { - Closeables.closeQuietly(input); + try { + input.close(); + } catch (IOException e) { + // Ignore + } LOG.warn("Failure in producing http content from location {}", location, failureCause); } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/worker/WorkerProgramRunner.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/worker/WorkerProgramRunner.java index 5a3cc0427568..86facfee0dc6 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/worker/WorkerProgramRunner.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/worker/WorkerProgramRunner.java @@ -160,10 +160,14 @@ public ProgramController run(Program program, ProgramOptions options) { ProgramController controller = new WorkerControllerServiceAdapter(worker, program.getId().run(runId)); - worker.start(); + worker.startAsync(); return controller; } catch (Throwable t) { - Closeables.closeQuietly(pluginInstantiator); + try { + pluginInstantiator.close(); + } catch (java.io.IOException e) { + // Ignore + } throw t; } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/workflow/DefaultProgramWorkflowRunner.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/workflow/DefaultProgramWorkflowRunner.java index 01ba0a9b1a00..e05f1cb32117 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/workflow/DefaultProgramWorkflowRunner.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/workflow/DefaultProgramWorkflowRunner.java @@ -166,7 +166,11 @@ private void runAndWait(ProgramRunner programRunner, Program program, ProgramOpt // If there is any exception when running the program, close the program to release resources. // Otherwise it will be released when the execution completed. programStateWriter.error(program.getId().run(runId), t); - Closeables.closeQuietly(closeable); + try { + closeable.close(); + } catch (IOException e) { + // Ignore + } throw t; } blockForCompletion(closeable, controller); @@ -210,7 +214,11 @@ public void init(ProgramController.State currentState, @Nullable Throwable cause @Override public void completed() { - Closeables.closeQuietly(closeable); + try { + closeable.close(); + } catch (IOException e) { + // Ignore + } Set fieldLineageOperations = new HashSet<>(); if (controller instanceof WorkflowDataProvider) { fieldLineageOperations.addAll( @@ -224,7 +232,11 @@ public void completed() { @Override public void killed() { - Closeables.closeQuietly(closeable); + try { + closeable.close(); + } catch (IOException e) { + // Ignore + } nodeStates.put(nodeId, new WorkflowNodeState(nodeId, NodeStatus.KILLED, controller.getRunId().getId(), null)); completion.set(null); @@ -232,7 +244,11 @@ public void killed() { @Override public void error(Throwable cause) { - Closeables.closeQuietly(closeable); + try { + closeable.close(); + } catch (IOException e) { + // Ignore + } nodeStates.put(nodeId, new WorkflowNodeState(nodeId, NodeStatus.FAILED, controller.getRunId().getId(), cause)); completion.setException(cause); @@ -271,7 +287,11 @@ private Closeable createCloseable(final ProgramRunner programRunner, final Progr return new Closeable() { @Override public void close() throws IOException { - Closeables.closeQuietly(program); + try { + program.close(); + } catch (IOException e) { + // Ignore + } closeProgramRunner(programRunner); } }; @@ -282,7 +302,11 @@ public void close() throws IOException { */ private void closeProgramRunner(ProgramRunner programRunner) { if (programRunner instanceof Closeable) { - Closeables.closeQuietly((Closeable) programRunner); + try { + ((Closeable) programRunner).close(); + } catch (IOException e) { + // Ignore + } } } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/workflow/WorkflowProgramController.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/workflow/WorkflowProgramController.java index bd2b44807ed7..c06db96effd5 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/workflow/WorkflowProgramController.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/workflow/WorkflowProgramController.java @@ -54,7 +54,7 @@ protected void doResume() throws Exception { @Override protected void doStop() throws Exception { - driver.stopAndWait(); + driver.stopAsync().awaitTerminated(); } @Override @@ -64,7 +64,7 @@ protected void doCommand(String name, Object value) throws Exception { private void startListen(Service service) { // Forward state changes from the given service to this controller. - service.addListener(new ServiceListenerAdapter() { + service.addListener(new Service.Listener() { @Override public void running() { diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/workflow/WorkflowProgramRunner.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/workflow/WorkflowProgramRunner.java index d20dea64f0ad..c115f5d2cfcd 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/workflow/WorkflowProgramRunner.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/workflow/WorkflowProgramRunner.java @@ -153,7 +153,7 @@ public ProgramController run(final Program program, final ProgramOptions options // service can be fully captured by the controller. ProgramController controller = new WorkflowProgramController(program.getId().run(runId), driver); - driver.start(); + driver.startAsync(); return controller; } catch (Exception e) { closeAllQuietly(closeables); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricProcessorService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricProcessorService.java index 97fafa95f819..968625801c4d 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricProcessorService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricProcessorService.java @@ -20,6 +20,7 @@ import com.google.common.util.concurrent.AbstractIdleService; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.Service.State; import com.google.inject.Inject; import com.google.inject.name.Named; import io.cdap.cdap.api.feature.FeatureFlagsProvider; @@ -86,6 +87,19 @@ public class AppFabricProcessorService extends AbstractIdleService { private CommonNettyHttpServiceFactory commonNettyHttpServiceFactory; private Cancellable cancelHttpService; private Set handlers; + private boolean auditLogSubscriberServiceStartedByMe = false; + private boolean provisioningServiceStartedByMe = false; + private boolean applicationLifecycleServiceStartedByMe = false; + private boolean bootstrapServiceStartedByMe = false; + private boolean programRuntimeServiceStartedByMe = false; + private boolean programNotificationSubscriberServiceStartedByMe = false; + private boolean programStopSubscriberServiceStartedByMe = false; + private boolean runRecordCorrectorServiceStartedByMe = false; + private boolean programRunStatusMonitorServiceStartedByMe = false; + private boolean coreSchedulerServiceStartedByMe = false; + private boolean runRecordCounterServiceStartedByMe = false; + private boolean runDataTimeToLiveServiceStartedByMe = false; + private boolean operationNotificationSubscriberServiceStartedByMe = false; /** * Construct the AppFabricProcessorService with service factory and cConf coming from guice injection. @@ -146,29 +160,108 @@ protected void startUp() throws Exception { Constants.Logging.COMPONENT_NAME, Service.APP_FABRIC_PROCESSOR)); LOG.info("Starting AppFabric processor service."); - List> futuresList = new ArrayList<>(); FeatureFlagsProvider featureFlagsProvider = new DefaultFeatureFlagsProvider(cConf); - // Only for RBAC instances - if (Feature.DATAPLANE_AUDIT_LOGGING.isEnabled(featureFlagsProvider) - && cConf.getBoolean(Constants.Security.ENABLED)) { - futuresList.add(auditLogSubscriberService.start()); - } - - futuresList.addAll(ImmutableList.of( - provisioningService.start(), - applicationLifecycleService.start(), - bootstrapService.start(), - programRuntimeService.start(), - programNotificationSubscriberService.start(), - programStopSubscriberService.start(), - runRecordCorrectorService.start(), - programRunStatusMonitorService.start(), - coreSchedulerService.start(), - runRecordCounterService.start(), - runDataTimeToLiveService.start(), - operationNotificationSubscriberService.start() - )); - Futures.allAsList(futuresList).get(); + boolean auditLoggingEnabled = Feature.DATAPLANE_AUDIT_LOGGING.isEnabled(featureFlagsProvider) + && cConf.getBoolean(Constants.Security.ENABLED); + + if (auditLoggingEnabled && auditLogSubscriberService.state() == State.NEW) { + auditLogSubscriberService.startAsync(); + auditLogSubscriberServiceStartedByMe = true; + } + + if (provisioningService.state() == State.NEW) { + provisioningService.startAsync(); + provisioningServiceStartedByMe = true; + } + if (applicationLifecycleService.state() == State.NEW) { + applicationLifecycleService.startAsync(); + applicationLifecycleServiceStartedByMe = true; + } + if (bootstrapService.state() == State.NEW) { + bootstrapService.startAsync(); + bootstrapServiceStartedByMe = true; + } + if (programRuntimeService.state() == State.NEW) { + programRuntimeService.startAsync(); + programRuntimeServiceStartedByMe = true; + } + if (programNotificationSubscriberService.state() == State.NEW) { + programNotificationSubscriberService.startAsync(); + programNotificationSubscriberServiceStartedByMe = true; + } + if (programStopSubscriberService.state() == State.NEW) { + programStopSubscriberService.startAsync(); + programStopSubscriberServiceStartedByMe = true; + } + if (runRecordCorrectorService.state() == State.NEW) { + runRecordCorrectorService.startAsync(); + runRecordCorrectorServiceStartedByMe = true; + } + if (programRunStatusMonitorService.state() == State.NEW) { + programRunStatusMonitorService.startAsync(); + programRunStatusMonitorServiceStartedByMe = true; + } + if (coreSchedulerService.state() == State.NEW) { + coreSchedulerService.startAsync(); + coreSchedulerServiceStartedByMe = true; + } + if (runRecordCounterService.state() == State.NEW) { + runRecordCounterService.startAsync(); + runRecordCounterServiceStartedByMe = true; + } + if (runDataTimeToLiveService.state() == State.NEW) { + runDataTimeToLiveService.startAsync(); + runDataTimeToLiveServiceStartedByMe = true; + } + if (operationNotificationSubscriberService.state() == State.NEW) { + operationNotificationSubscriberService.startAsync(); + operationNotificationSubscriberServiceStartedByMe = true; + } + + if (auditLoggingEnabled + && (auditLogSubscriberServiceStartedByMe + || auditLogSubscriberService.state() == State.STARTING)) { + auditLogSubscriberService.awaitRunning(); + } + if (provisioningServiceStartedByMe || provisioningService.state() == State.STARTING) { + provisioningService.awaitRunning(); + } + if (applicationLifecycleServiceStartedByMe || applicationLifecycleService.state() == State.STARTING) { + applicationLifecycleService.awaitRunning(); + } + if (bootstrapServiceStartedByMe || bootstrapService.state() == State.STARTING) { + bootstrapService.awaitRunning(); + } + if (programRuntimeServiceStartedByMe || programRuntimeService.state() == State.STARTING) { + programRuntimeService.awaitRunning(); + } + if (programNotificationSubscriberServiceStartedByMe + || programNotificationSubscriberService.state() == State.STARTING) { + programNotificationSubscriberService.awaitRunning(); + } + if (programStopSubscriberServiceStartedByMe || programStopSubscriberService.state() == State.STARTING) { + programStopSubscriberService.awaitRunning(); + } + if (runRecordCorrectorServiceStartedByMe || runRecordCorrectorService.state() == State.STARTING) { + runRecordCorrectorService.awaitRunning(); + } + if (programRunStatusMonitorServiceStartedByMe + || programRunStatusMonitorService.state() == State.STARTING) { + programRunStatusMonitorService.awaitRunning(); + } + if (coreSchedulerServiceStartedByMe || coreSchedulerService.state() == State.STARTING) { + coreSchedulerService.awaitRunning(); + } + if (runRecordCounterServiceStartedByMe || runRecordCounterService.state() == State.STARTING) { + runRecordCounterService.awaitRunning(); + } + if (runDataTimeToLiveServiceStartedByMe || runDataTimeToLiveService.state() == State.STARTING) { + runDataTimeToLiveService.awaitRunning(); + } + if (operationNotificationSubscriberServiceStartedByMe + || operationNotificationSubscriberService.state() == State.STARTING) { + operationNotificationSubscriberService.awaitRunning(); + } // Run http service on random port NettyHttpService.Builder httpServiceBuilder = commonNettyHttpServiceFactory @@ -196,20 +289,46 @@ protected void startUp() throws Exception { protected void shutDown() throws Exception { LOG.info("Stopping AppFabric processor service."); cancelHttpService.cancel(); - coreSchedulerService.stopAndWait(); - bootstrapService.stopAndWait(); - systemAppManagementService.stopAndWait(); - programRuntimeService.stopAndWait(); - applicationLifecycleService.stopAndWait(); - programNotificationSubscriberService.stopAndWait(); - programStopSubscriberService.stopAndWait(); - runRecordCorrectorService.stopAndWait(); - programRunStatusMonitorService.stopAndWait(); - provisioningService.stopAndWait(); - runRecordCounterService.stopAndWait(); - runDataTimeToLiveService.stopAndWait(); - operationNotificationSubscriberService.stopAndWait(); - auditLogSubscriberService.stopAndWait(); + if (coreSchedulerServiceStartedByMe) { + coreSchedulerService.stopAsync().awaitTerminated(); + } + if (bootstrapServiceStartedByMe) { + bootstrapService.stopAsync().awaitTerminated(); + } + systemAppManagementService.stopAsync().awaitTerminated(); + if (programRuntimeServiceStartedByMe) { + programRuntimeService.stopAsync().awaitTerminated(); + } + if (applicationLifecycleServiceStartedByMe) { + applicationLifecycleService.stopAsync().awaitTerminated(); + } + if (programNotificationSubscriberServiceStartedByMe) { + programNotificationSubscriberService.stopAsync().awaitTerminated(); + } + if (programStopSubscriberServiceStartedByMe) { + programStopSubscriberService.stopAsync().awaitTerminated(); + } + if (runRecordCorrectorServiceStartedByMe) { + runRecordCorrectorService.stopAsync().awaitTerminated(); + } + if (programRunStatusMonitorServiceStartedByMe) { + programRunStatusMonitorService.stopAsync().awaitTerminated(); + } + if (provisioningServiceStartedByMe) { + provisioningService.stopAsync().awaitTerminated(); + } + if (runRecordCounterServiceStartedByMe) { + runRecordCounterService.stopAsync().awaitTerminated(); + } + if (runDataTimeToLiveServiceStartedByMe) { + runDataTimeToLiveService.stopAsync().awaitTerminated(); + } + if (operationNotificationSubscriberServiceStartedByMe) { + operationNotificationSubscriberService.stopAsync().awaitTerminated(); + } + if (auditLogSubscriberServiceStartedByMe) { + auditLogSubscriberService.stopAsync().awaitTerminated(); + } LOG.info("AppFabric processor service stopped."); } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricServer.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricServer.java index bf6d0eb9d423..ac00679a16f5 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricServer.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricServer.java @@ -20,6 +20,7 @@ import com.google.common.util.concurrent.AbstractIdleService; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.Service; import com.google.inject.Inject; import com.google.inject.name.Named; import io.cdap.cdap.api.feature.FeatureFlagsProvider; @@ -92,6 +93,13 @@ public class AppFabricServer extends AbstractIdleService { private Set handlers; private MetricsCollectionService metricsCollectionService; private CommonNettyHttpServiceFactory commonNettyHttpServiceFactory; + private boolean namespaceCredentialProviderServiceStartedByMe = false; + private boolean provisioningServiceStartedByMe = false; + private boolean applicationLifecycleServiceStartedByMe = false; + private boolean bootstrapServiceStartedByMe = false; + private boolean credentialProviderServiceStartedByMe = false; + private boolean sourceControlOperationRunnerStartedByMe = false; + private boolean repositoryCleanupServiceStartedByMe = false; /** * Construct the AppFabricServer with service factory and cConf coming from guice @@ -143,20 +151,62 @@ protected void startUp() throws Exception { new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, Constants.Service.APP_FABRIC_HTTP)); - List> futuresList = new ArrayList<>(); FeatureFlagsProvider featureFlagsProvider = new DefaultFeatureFlagsProvider(cConf); - if (Feature.NAMESPACED_SERVICE_ACCOUNTS.isEnabled(featureFlagsProvider)) { - futuresList.add(namespaceCredentialProviderService.start()); + boolean namespacedServiceAccountsEnabled = Feature.NAMESPACED_SERVICE_ACCOUNTS.isEnabled(featureFlagsProvider); + + if (namespacedServiceAccountsEnabled && namespaceCredentialProviderService.state() == Service.State.NEW) { + namespaceCredentialProviderService.startAsync(); + namespaceCredentialProviderServiceStartedByMe = true; + } + + if (provisioningService.state() == Service.State.NEW) { + provisioningService.startAsync(); + provisioningServiceStartedByMe = true; + } + if (applicationLifecycleService.state() == Service.State.NEW) { + applicationLifecycleService.startAsync(); + applicationLifecycleServiceStartedByMe = true; + } + if (bootstrapService.state() == Service.State.NEW) { + bootstrapService.startAsync(); + bootstrapServiceStartedByMe = true; + } + if (credentialProviderService.state() == Service.State.NEW) { + credentialProviderService.startAsync(); + credentialProviderServiceStartedByMe = true; + } + if (sourceControlOperationRunner.state() == Service.State.NEW) { + sourceControlOperationRunner.startAsync(); + sourceControlOperationRunnerStartedByMe = true; + } + if (repositoryCleanupService.state() == Service.State.NEW) { + repositoryCleanupService.startAsync(); + repositoryCleanupServiceStartedByMe = true; + } + + if (namespacedServiceAccountsEnabled + && (namespaceCredentialProviderServiceStartedByMe + || namespaceCredentialProviderService.state() == Service.State.STARTING)) { + namespaceCredentialProviderService.awaitRunning(); + } + if (provisioningServiceStartedByMe || provisioningService.state() == Service.State.STARTING) { + provisioningService.awaitRunning(); + } + if (applicationLifecycleServiceStartedByMe || applicationLifecycleService.state() == Service.State.STARTING) { + applicationLifecycleService.awaitRunning(); + } + if (bootstrapServiceStartedByMe || bootstrapService.state() == Service.State.STARTING) { + bootstrapService.awaitRunning(); + } + if (credentialProviderServiceStartedByMe || credentialProviderService.state() == Service.State.STARTING) { + credentialProviderService.awaitRunning(); + } + if (sourceControlOperationRunnerStartedByMe || sourceControlOperationRunner.state() == Service.State.STARTING) { + sourceControlOperationRunner.awaitRunning(); + } + if (repositoryCleanupServiceStartedByMe || repositoryCleanupService.state() == Service.State.STARTING) { + repositoryCleanupService.awaitRunning(); } - futuresList.addAll(ImmutableList.of( - provisioningService.start(), - applicationLifecycleService.start(), - bootstrapService.start(), - credentialProviderService.start(), - sourceControlOperationRunner.start(), - repositoryCleanupService.start() - )); - Futures.allAsList(futuresList).get(); // Create handler hooks List handlerHooks = handlerHookNames.stream() @@ -212,13 +262,27 @@ protected void startUp() throws Exception { @Override protected void shutDown() throws Exception { cancelHttpService.cancel(); - applicationLifecycleService.stopAndWait(); - bootstrapService.stopAndWait(); - provisioningService.stopAndWait(); - sourceControlOperationRunner.stopAndWait(); - repositoryCleanupService.stopAndWait(); - credentialProviderService.stopAndWait(); - namespaceCredentialProviderService.stopAndWait(); + if (applicationLifecycleServiceStartedByMe) { + applicationLifecycleService.stopAsync().awaitTerminated(); + } + if (bootstrapServiceStartedByMe) { + bootstrapService.stopAsync().awaitTerminated(); + } + if (provisioningServiceStartedByMe) { + provisioningService.stopAsync().awaitTerminated(); + } + if (sourceControlOperationRunnerStartedByMe) { + sourceControlOperationRunner.stopAsync().awaitTerminated(); + } + if (repositoryCleanupServiceStartedByMe) { + repositoryCleanupService.stopAsync().awaitTerminated(); + } + if (credentialProviderServiceStartedByMe) { + credentialProviderService.stopAsync().awaitTerminated(); + } + if (namespaceCredentialProviderServiceStartedByMe) { + namespaceCredentialProviderService.stopAsync().awaitTerminated(); + } } private Cancellable startHttpService(NettyHttpService httpService) throws Exception { diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ProgramNotificationSubscriberService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ProgramNotificationSubscriberService.java index 4b5b377fa234..74473d995c6c 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ProgramNotificationSubscriberService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ProgramNotificationSubscriberService.java @@ -162,7 +162,7 @@ protected void startUp() throws Exception { .forEach(i -> children.add(createChildService("program.status." + i, topicPrefix + i))); } delegate = new CompositeService(children); - delegate.startAndWait(); + delegate.startAsync().awaitRunning(); // Explicitly emit both launching and running counts on startup. emitFlowControlMetrics(); } @@ -218,7 +218,7 @@ private void restoreActiveRuns() { @Override protected void shutDown() throws Exception { - delegate.stopAndWait(); + delegate.stopAsync().awaitTerminated(); } @Inject(optional = true) diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/ApplicationMeta.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/ApplicationMeta.java index 1db650299aeb..cc71ad2e7777 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/ApplicationMeta.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/ApplicationMeta.java @@ -16,8 +16,9 @@ package io.cdap.cdap.internal.app.store; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import io.cdap.cdap.api.app.ApplicationSpecification; +import java.util.Objects; import io.cdap.cdap.internal.app.ApplicationSpecificationAdapter; import io.cdap.cdap.proto.artifact.ChangeDetail; import io.cdap.cdap.proto.sourcecontrol.SourceControlMeta; @@ -69,7 +70,7 @@ public SourceControlMeta getSourceControlMeta() { @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("id", id) .add("spec", ADAPTER.toJson(spec)) .add("change", change) @@ -86,14 +87,14 @@ public boolean equals(Object o) { return false; } ApplicationMeta that = (ApplicationMeta) o; - return Objects.equal(id, that.id) - && Objects.equal(spec, that.spec) - && Objects.equal(change, that.change) - && Objects.equal(sourceControlMeta, that.sourceControlMeta); + return Objects.equals(id, that.id) + && Objects.equals(spec, that.spec) + && Objects.equals(change, that.change) + && Objects.equals(sourceControlMeta, that.sourceControlMeta); } @Override public int hashCode() { - return Objects.hashCode(id, spec, change, sourceControlMeta); + return Objects.hash(id, spec, change, sourceControlMeta); } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/DefaultStore.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/DefaultStore.java index 57fb3c102cef..b88798bc4146 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/DefaultStore.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/DefaultStore.java @@ -19,7 +19,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; import com.google.common.collect.MapDifference; import com.google.common.collect.Maps; import com.google.inject.Inject; @@ -638,7 +637,7 @@ public List getDeletedProgramSpecifications(ApplicationRef return getAppMetadataStore(context).getLatest(appRef); }); - List deletedProgramSpecs = Lists.newArrayList(); + List deletedProgramSpecs = new ArrayList<>(); if (existing != null) { ApplicationSpecification existingAppSpec = existing.getSpec(); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/adapters/PluginKey.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/adapters/PluginKey.java index 70f53476f686..3a938dd0159a 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/adapters/PluginKey.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/adapters/PluginKey.java @@ -16,7 +16,8 @@ package io.cdap.cdap.internal.app.store.adapters; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; +import java.util.Objects; /** * Unique key for identifying a plugin. Used for look up and caching. @@ -106,13 +107,13 @@ public boolean equals(Object o) { return false; } PluginKey pluginKey = (PluginKey) o; - return Objects.equal(parentName, pluginKey.parentName) - && Objects.equal(parentNamespace, pluginKey.parentNamespace) - && Objects.equal(artifactName, pluginKey.artifactName) - && Objects.equal(artifactNamespace, pluginKey.artifactNamespace) - && Objects.equal(artifactVersion, pluginKey.artifactVersion) - && Objects.equal(pluginType, pluginKey.pluginType) - && Objects.equal(pluginName, pluginKey.pluginName); + return Objects.equals(parentName, pluginKey.parentName) + && Objects.equals(parentNamespace, pluginKey.parentNamespace) + && Objects.equals(artifactName, pluginKey.artifactName) + && Objects.equals(artifactNamespace, pluginKey.artifactNamespace) + && Objects.equals(artifactVersion, pluginKey.artifactVersion) + && Objects.equals(pluginType, pluginKey.pluginType) + && Objects.equals(pluginName, pluginKey.pluginName); } /** @@ -120,7 +121,7 @@ public boolean equals(Object o) { */ @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("parentName", parentName) .add("parentNamespace", parentNamespace) .add("artifactName", artifactName) @@ -136,7 +137,7 @@ public String toString() { */ @Override public int hashCode() { - return Objects.hashCode(parentName, parentNamespace, artifactName, artifactNamespace, + return Objects.hash(parentName, parentNamespace, artifactName, artifactNamespace, artifactVersion, pluginType, pluginName); } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/TaskWorkerService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/TaskWorkerService.java index 294cbef60e50..c3a22e98526f 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/TaskWorkerService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/TaskWorkerService.java @@ -126,7 +126,7 @@ private void stopService(String className) { * based on number of requests per particular class, * the service gets stopped. */ - stop(); + stopAsync(); } @VisibleForTesting diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/TaskWorkerTwillRunnable.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/TaskWorkerTwillRunnable.java index 527c158a8295..191649f9df77 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/TaskWorkerTwillRunnable.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/TaskWorkerTwillRunnable.java @@ -147,7 +147,7 @@ public void initialize(TwillContext context) { @Override public void run() { CompletableFuture future = new CompletableFuture<>(); - taskWorker.addListener(new ServiceListenerAdapter() { + taskWorker.addListener(new Service.Listener() { @Override public void terminated(Service.State from) { future.complete(from); @@ -160,7 +160,7 @@ public void failed(Service.State from, Throwable failure) { }, Threads.SAME_THREAD_EXECUTOR); LOG.debug("Starting task worker"); - taskWorker.start(); + taskWorker.startAsync(); try { Uninterruptibles.getUninterruptibly(future); @@ -173,9 +173,9 @@ public void failed(Service.State from, Throwable failure) { @Override public void stop() { LOG.info("Stopping task worker"); - Optional.ofNullable(metricsCollectionService).map(MetricsCollectionService::stop); + Optional.ofNullable(metricsCollectionService).ifPresent(Service::stopAsync); if (taskWorker != null) { - taskWorker.stop(); + taskWorker.stopAsync(); } } @@ -203,7 +203,7 @@ private void doInitialize(TwillContext context) throws Exception { logAppenderInitializer.initialize(); metricsCollectionService = injector.getInstance(MetricsCollectionService.class); - metricsCollectionService.startAndWait(); + metricsCollectionService.startAsync().awaitRunning(); LoggingContext loggingContext = new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/sidecar/ArtifactLocalizerTwillRunnable.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/sidecar/ArtifactLocalizerTwillRunnable.java index 67091c898206..a77ae345f806 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/sidecar/ArtifactLocalizerTwillRunnable.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/sidecar/ArtifactLocalizerTwillRunnable.java @@ -167,7 +167,7 @@ public void initialize(TwillContext context) { @Override public void run() { CompletableFuture future = new CompletableFuture<>(); - artifactLocalizerService.addListener(new ServiceListenerAdapter() { + artifactLocalizerService.addListener(new Service.Listener() { @Override public void terminated(Service.State from) { future.complete(from); @@ -180,7 +180,7 @@ public void failed(Service.State from, Throwable failure) { }, Threads.SAME_THREAD_EXECUTOR); LOG.debug("Starting artifact localizer"); - artifactLocalizerService.start(); + artifactLocalizerService.startAsync(); try { Uninterruptibles.getUninterruptibly(future); @@ -191,13 +191,13 @@ public void failed(Service.State from, Throwable failure) { @Override public void stop() { - artifactLocalizerService.stop(); + artifactLocalizerService.stopAsync(); } @Override public void destroy() { try { - tokenManager.stopAndWait(); + tokenManager.stopAsync().awaitTerminated(); } finally { logAppenderInitializer.close(); } @@ -225,7 +225,7 @@ void doInitialize() throws Exception { LoggingContextAccessor.setLoggingContext(loggingContext); tokenManager = injector.getInstance(TokenManager.class); - tokenManager.startAndWait(); + tokenManager.startAsync().awaitRunning(); artifactLocalizerService = injector.getInstance(ArtifactLocalizerService.class); } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/system/SystemWorkerService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/system/SystemWorkerService.java index c374f8e77da5..d576b2c46c6e 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/system/SystemWorkerService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/system/SystemWorkerService.java @@ -106,7 +106,7 @@ public void modify(ChannelPipeline pipeline) { @Override protected void startUp() throws Exception { LOG.debug("Starting SystemWorkerService"); - tokenManager.startAndWait(); + tokenManager.startAsync().awaitRunning(); provisioningService.initializeProvisionersAndExecutors(); twillRunnerService.start(); remoteTwillRunnerService.start(); @@ -121,7 +121,7 @@ protected void startUp() throws Exception { @Override protected void shutDown() throws Exception { LOG.debug("Shutting down SystemWorkerService"); - tokenManager.stop(); + tokenManager.stopAsync().awaitTerminated(); twillRunnerService.stop(); remoteTwillRunnerService.stop(); httpService.stop(1, 2, TimeUnit.SECONDS); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/system/SystemWorkerTwillRunnable.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/system/SystemWorkerTwillRunnable.java index 18c27da9c734..79a4a763f394 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/system/SystemWorkerTwillRunnable.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/system/SystemWorkerTwillRunnable.java @@ -273,7 +273,7 @@ public void initialize(TwillContext context) { @Override public void run() { CompletableFuture future = new CompletableFuture<>(); - systemWorker.addListener(new ServiceListenerAdapter() { + systemWorker.addListener(new Service.Listener() { @Override public void terminated(Service.State from) { future.complete(from); @@ -286,9 +286,9 @@ public void failed(Service.State from, Throwable failure) { }, Threads.SAME_THREAD_EXECUTOR); LOG.debug("Starting system worker"); - systemWorker.start(); + systemWorker.startAsync(); if (artifactLocalizerService != null) { - artifactLocalizerService.start(); + artifactLocalizerService.startAsync(); } try { @@ -302,12 +302,12 @@ public void failed(Service.State from, Throwable failure) { @Override public void stop() { LOG.info("Stopping system worker"); - Optional.ofNullable(metricsCollectionService).map(MetricsCollectionService::stop); + Optional.ofNullable(metricsCollectionService).ifPresent(Service::stopAsync); if (systemWorker != null) { - systemWorker.stop(); + systemWorker.stopAsync(); } if (artifactLocalizerService != null) { - artifactLocalizerService.stop(); + artifactLocalizerService.stopAsync(); } } @@ -342,7 +342,7 @@ void doInitialize() throws Exception { logAppenderInitializer.initialize(); metricsCollectionService = injector.getInstance(MetricsCollectionService.class); - metricsCollectionService.startAndWait(); + metricsCollectionService.startAsync().awaitRunning(); LoggingContext loggingContext = new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/bootstrap/BootstrapService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/bootstrap/BootstrapService.java index 70f95fe93b77..fa4a8bda1af4 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/bootstrap/BootstrapService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/bootstrap/BootstrapService.java @@ -108,13 +108,13 @@ protected void startUp() { // LOAD_SYSTEM_ARTIFACT step. // TODO(CDAP-16243): Find better way to add dependency between BootStrapService and SystemAppManagementService. try { - this.systemAppManagementService.start(); + this.systemAppManagementService.startAsync(); } catch (Exception e) { LOG.info("SystemAppManagementService could not start due to exception.", e); } // TODO - Move this back to AppFabricServer once CDAP-17578 is fixed - systemProgramManagementService.start(); - capabilityManagementService.start(); + systemProgramManagementService.startAsync(); + capabilityManagementService.startAsync(); }); LOG.info("Started {}", getClass().getSimpleName()); } @@ -125,9 +125,9 @@ protected void shutDown() throws Exception { // Shutdown the executor, which will issue an interrupt to the running thread. // There is only a single daemon thread, so no need to wait for termination executorService.shutdownNow(); - this.systemAppManagementService.stopAndWait(); - capabilityManagementService.stopAndWait(); - systemProgramManagementService.stopAndWait(); + this.systemAppManagementService.stopAsync().awaitTerminated(); + capabilityManagementService.stopAsync().awaitTerminated(); + systemProgramManagementService.stopAsync().awaitTerminated(); LOG.info("Stopped {}", getClass().getSimpleName()); } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/capability/autoinstall/HubPackage.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/capability/autoinstall/HubPackage.java index acfca6459846..f8ed0d4903da 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/capability/autoinstall/HubPackage.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/capability/autoinstall/HubPackage.java @@ -180,7 +180,11 @@ public boolean onReceived(ByteBuffer buffer) { @Override public void onFinished() { - Closeables.closeQuietly(channel); + try { + channel.close(); + } catch (IOException e) { + // Ignore + } } }).build(); HttpClients.executeStreamingRequest(request); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/events/EventSubscriberManager.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/events/EventSubscriberManager.java index f41539a1db32..d73d894bac9b 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/events/EventSubscriberManager.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/events/EventSubscriberManager.java @@ -50,7 +50,7 @@ protected void startUp() throws Exception { // Initialize the event subscribers with all the event readers provided by provider try { eventSubscriber.initialize(); - eventSubscriber.startAndWait(); + eventSubscriber.startAsync().awaitRunning(); LOG.info("Successfully initialized eventSubscriber: {}", eventSubscriber); } catch (Exception e) { @@ -67,7 +67,7 @@ protected void shutDown() throws Exception { } eventSubscribers.forEach(eventSubscriber -> { try { - eventSubscriber.stopAndWait(); + eventSubscriber.stopAsync().awaitTerminated(); } catch (Exception e) { LOG.error("Failed to stop subscriber", e); } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/events/ProgramStatusEventPublisher.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/events/ProgramStatusEventPublisher.java index ff2461fd8387..32fd1477ffc1 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/events/ProgramStatusEventPublisher.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/events/ProgramStatusEventPublisher.java @@ -108,12 +108,12 @@ public void initialize(Collection eventWriters) { @Override public void startPublish() { - super.startAndWait(); + super.startAsync().awaitRunning(); } @Override public void stopPublish() { - this.stop(); + this.stopAsync(); } @Override diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/operation/InMemoryOperationController.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/operation/InMemoryOperationController.java index 17973486d2a4..e618f9c9e0b7 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/operation/InMemoryOperationController.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/operation/InMemoryOperationController.java @@ -52,7 +52,7 @@ public class InMemoryOperationController implements @Override public ListenableFuture stop() { LOG.trace("Stopping operation {}", runId); - driver.stop(); + driver.stopAsync(); return completionFuture; } @@ -62,7 +62,7 @@ public ListenableFuture complete() { } private void startListen(Service service) { - service.addListener(new ServiceListenerAdapter() { + service.addListener(new Service.Listener() { @Override public void running() { statePublisher.publishRunning(runId); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/operation/InMemoryOperationRunner.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/operation/InMemoryOperationRunner.java index 513b160b5e3b..6946cd15d9d2 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/operation/InMemoryOperationRunner.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/operation/InMemoryOperationRunner.java @@ -45,7 +45,7 @@ public OperationController run(OperationRunDetail detail) throws IllegalStateExc OperationDriver driver = new OperationDriver(createOperation(detail), context); OperationController controller = new InMemoryOperationController(context.getRunId(), statePublisher, driver); - driver.start(); + driver.startAsync(); return controller; } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/operation/OperationNotificationSubscriberService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/operation/OperationNotificationSubscriberService.java index ae20897617f2..afc28ee4af41 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/operation/OperationNotificationSubscriberService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/operation/OperationNotificationSubscriberService.java @@ -93,7 +93,7 @@ protected void startUp() throws Exception { .forEach(i -> children.add(createChildService("operation.status." + i, topicPrefix + i))); delegate = new CompositeService(children); - delegate.startAndWait(); + delegate.startAsync().awaitRunning(); } // Sends STARTING notification for all STARTING operations @@ -130,7 +130,7 @@ private void processStoppingOperations(StructuredTableContext context) throws Ex @Override protected void shutDown() throws Exception { - delegate.stopAndWait(); + delegate.stopAsync().awaitTerminated(); } private OperationNotificationSingleTopicSubscriberService createChildService( diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/master/environment/k8s/AbstractServiceMain.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/master/environment/k8s/AbstractServiceMain.java index 8f4996468171..599039b50b46 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/master/environment/k8s/AbstractServiceMain.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/master/environment/k8s/AbstractServiceMain.java @@ -226,7 +226,7 @@ public final void start() { LOG.info("Starting all services for {}", getClass().getName()); for (Service service : services) { LOG.info("Starting service {} for {}", service, getClass().getName()); - service.startAndWait(); + service.startAsync().awaitRunning(); } LOG.info("All services for {} started", getClass().getName()); } @@ -237,7 +237,7 @@ public final void stop() { for (Service service : Lists.reverse(services)) { LOG.info("Stopping service {} for {}", service, getClass().getName()); try { - service.stopAndWait(); + service.stopAsync().awaitTerminated(); } catch (Exception e) { // Catch and log exception on stopping to make sure each service has a chance to stop LOG.warn("Exception raised when stopping service {} for {}", service, getClass().getName(), diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/master/environment/k8s/MasterEnvironmentMain.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/master/environment/k8s/MasterEnvironmentMain.java index 71bef4a011c9..77c06c130c68 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/master/environment/k8s/MasterEnvironmentMain.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/master/environment/k8s/MasterEnvironmentMain.java @@ -159,7 +159,9 @@ public static void doMain(String[] args) throws Exception { runnable.stop(); Uninterruptibles.awaitUninterruptibly(shutdownLatch, 30, TimeUnit.SECONDS); } - Optional.ofNullable(tokenManager).ifPresent(TokenManager::stopAndWait); + if (tokenManager != null) { + tokenManager.stopAsync().awaitTerminated(); + } })); runnable.run(runnableArgs); completed.set(true); @@ -192,7 +194,7 @@ private static InternalAuthenticator getInternalAuthenticator(CConfiguration cCo new AuthenticationContextModules().getMasterModule()); if (cConf.getBoolean(Constants.Security.INTERNAL_AUTH_ENABLED)) { tokenManager = injector.getInstance(TokenManager.class); - tokenManager.startAndWait(); + tokenManager.startAsync().awaitRunning(); } } else { // cdap-secret is NOT mounted, use worker authentication context diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/metadata/MetadataValidator.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/metadata/MetadataValidator.java index 40dd0c34e134..5cc856f794c7 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/metadata/MetadataValidator.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/metadata/MetadataValidator.java @@ -42,7 +42,7 @@ public class MetadataValidator { .or(CharMatcher.inRange('0', '9')) .or(CharMatcher.is('_')) .or(CharMatcher.is('-')) - .or(CharMatcher.WHITESPACE); + .or(CharMatcher.whitespace()); private final int maxCharacters; diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/scheduler/ConstraintCheckerService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/scheduler/ConstraintCheckerService.java index e6f12fcbff06..c801a7d88d8f 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/scheduler/ConstraintCheckerService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/scheduler/ConstraintCheckerService.java @@ -192,9 +192,9 @@ private boolean checkJobConstraints(JobQueue jobQueue) throws IOException { boolean emptyScan = true; try (CloseableIterator jobQueueIter = jobQueue.getJobs(partition, lastConsumed)) { - Stopwatch stopWatch = new Stopwatch().start(); + Stopwatch stopWatch = Stopwatch.createStarted(); // limit the batches of the scan to 1000ms - while (!stopping && stopWatch.elapsedMillis() < 1000) { + while (!stopping && stopWatch.elapsed(TimeUnit.MILLISECONDS) < 1000) { if (!jobQueueIter.hasNext()) { lastConsumed = null; return emptyScan; diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/scheduler/CoreSchedulerService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/scheduler/CoreSchedulerService.java index 51507d596861..6c6b1a8b8f3b 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/scheduler/CoreSchedulerService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/scheduler/CoreSchedulerService.java @@ -115,25 +115,25 @@ public class CoreSchedulerService extends AbstractIdleService implements Schedul this.internalService = new RetryOnStartFailureService(() -> new AbstractIdleService() { @Override - protected Executor executor(final State state) { - return command -> new Thread(command, "core scheduler service " + state).start(); + protected java.util.concurrent.Executor executor() { + return command -> new Thread(command, "core scheduler service " + state()).start(); } @Override protected void startUp() { - timeSchedulerService.startAndWait(); + timeSchedulerService.startAsync().awaitRunning(); cleanupJobs(); - constraintCheckerService.startAndWait(); - scheduleNotificationSubscriberService.startAndWait(); + constraintCheckerService.startAsync().awaitRunning(); + scheduleNotificationSubscriberService.startAsync().awaitRunning(); startedLatch.countDown(); LOG.info("Started core scheduler service."); } @Override protected void shutDown() { - scheduleNotificationSubscriberService.stopAndWait(); - constraintCheckerService.stopAndWait(); - timeSchedulerService.stopAndWait(); + scheduleNotificationSubscriberService.stopAsync().awaitTerminated(); + constraintCheckerService.stopAsync().awaitTerminated(); + timeSchedulerService.stopAsync().awaitTerminated(); LOG.info("Stopped core scheduler service."); } }, io.cdap.cdap.common.service.RetryStrategies.exponentialDelay(200, 5000, @@ -203,12 +203,12 @@ private void checkStarted() { @Override protected void startUp() throws Exception { - internalService.startAndWait(); + internalService.startAsync().awaitRunning(); } @Override protected void shutDown() throws Exception { - internalService.stopAndWait(); + internalService.stopAsync().awaitTerminated(); } @Override diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/scheduler/ProgramScheduleService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/scheduler/ProgramScheduleService.java index 751bb8d6d0dd..8c34573a04c3 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/scheduler/ProgramScheduleService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/scheduler/ProgramScheduleService.java @@ -16,7 +16,7 @@ package io.cdap.cdap.scheduler; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.inject.Inject; import io.cdap.cdap.api.ProgramStatus; import io.cdap.cdap.api.schedule.Trigger; @@ -153,13 +153,13 @@ public void update(ScheduleId scheduleId, ScheduleDetail scheduleDetail) throws ApplicationPermission.EXECUTE); ProgramSchedule existing = scheduler.getSchedule(scheduleId); - String description = Objects.firstNonNull(scheduleDetail.getDescription(), + String description = MoreObjects.firstNonNull(scheduleDetail.getDescription(), existing.getDescription()); ProgramId programId = scheduleDetail.getProgram() == null ? existing.getProgramId() : existing.getProgramId().getParent().program( scheduleDetail.getProgram().getProgramType() == null ? existing.getProgramId().getType() : ProgramType.valueOfSchedulableType(scheduleDetail.getProgram().getProgramType()), - Objects.firstNonNull(scheduleDetail.getProgram().getProgramName(), + MoreObjects.firstNonNull(scheduleDetail.getProgram().getProgramName(), existing.getProgramId().getProgram())); if (!programId.equals(existing.getProgramId())) { throw new BadRequestException( @@ -167,12 +167,12 @@ public void update(ScheduleId scheduleId, ScheduleDetail scheduleDetail) throws + "To change the program in a schedule, please delete the schedule and create a new one.", existing.getName(), existing.getProgramId().toString())); } - Map properties = Objects.firstNonNull(scheduleDetail.getProperties(), + Map properties = MoreObjects.firstNonNull(scheduleDetail.getProperties(), existing.getProperties()); - Trigger trigger = Objects.firstNonNull(scheduleDetail.getTrigger(), existing.getTrigger()); + Trigger trigger = MoreObjects.firstNonNull(scheduleDetail.getTrigger(), existing.getTrigger()); List constraints = - Objects.firstNonNull(scheduleDetail.getConstraints(), existing.getConstraints()); - Long timeoutMillis = Objects.firstNonNull(scheduleDetail.getTimeoutMillis(), + MoreObjects.firstNonNull(scheduleDetail.getConstraints(), existing.getConstraints()); + Long timeoutMillis = MoreObjects.firstNonNull(scheduleDetail.getTimeoutMillis(), existing.getTimeoutMillis()); ProgramSchedule updatedSchedule = new ProgramSchedule(existing.getName(), description, programId, properties, diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/scheduler/ScheduleNotificationSubscriberService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/scheduler/ScheduleNotificationSubscriberService.java index 8790002ce660..8c3e83aeee8b 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/scheduler/ScheduleNotificationSubscriberService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/scheduler/ScheduleNotificationSubscriberService.java @@ -96,23 +96,25 @@ protected void startUp() throws Exception { 1, Threads.createDaemonThreadFactory("scheduler-notification-subscriber-%d")); // Start all subscriber services. All of them has no-op in start, so they shouldn't fail. - Futures.successfulAsList( - subscriberServices.stream().map(Service::start).collect(Collectors.toList())).get(); + for (Service service : subscriberServices) { + service.startAsync(); + } + for (Service service : subscriberServices) { + service.awaitRunning(); + } } @Override protected void shutDown() throws Exception { - // This never throw - Futures.successfulAsList( - subscriberServices.stream().map(Service::stop).collect(Collectors.toList())).get(); + for (Service service : subscriberServices) { + service.stopAsync(); + } for (Service service : subscriberServices) { - // The service must have been stopped, and calling stop again will just return immediate with the - // future that carries the stop state. try { - service.stop().get(); - } catch (ExecutionException e) { - LOG.warn("Exception raised when stopping service {}", service, e.getCause()); + service.awaitTerminated(); + } catch (IllegalStateException e) { + LOG.warn("Exception raised when stopping service {}", service, e); } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/security/auth/AuditLogSubscriberService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/security/auth/AuditLogSubscriberService.java index d18d0bc2d55a..539d764f1353 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/security/auth/AuditLogSubscriberService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/security/auth/AuditLogSubscriberService.java @@ -69,13 +69,13 @@ protected void startUp() throws Exception { .forEach(i -> children.add(createChildService(topicPrefix + i))); delegate = new CompositeService(children); - delegate.startAndWait(); + delegate.startAsync().awaitRunning(); LOG.debug("Started Audit Log subscriber service for {} partitions.", numPartitions); } @Override protected void shutDown() throws Exception { - delegate.stopAndWait(); + delegate.stopAsync().awaitTerminated(); } private AuditLogSingleTopicSubscriberService createChildService(String topicName) { diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/security/hive/JobHistoryServerTokenUtils.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/security/hive/JobHistoryServerTokenUtils.java index 3bb1f4746914..4af6fd301d46 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/security/hive/JobHistoryServerTokenUtils.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/security/hive/JobHistoryServerTokenUtils.java @@ -65,7 +65,7 @@ public static Credentials obtainToken(Configuration configuration, Credentials c GetDelegationTokenRequest request = new GetDelegationTokenRequestPBImpl(); request.setRenewer(YarnUtils.getYarnTokenRenewer(configuration)); - InetSocketAddress address = new InetSocketAddress(hostAndPort.getHostText(), + InetSocketAddress address = new InetSocketAddress(hostAndPort.getHost(), hostAndPort.getPort()); Token token = ConverterUtils.convertFromYarn(hsProxy.getDelegationToken(request).getDelegationToken(), diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/app/mapreduce/LocalMRJobInfoFetcherTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/app/mapreduce/LocalMRJobInfoFetcherTest.java index 6a05608c3b84..b1dfd5ab59ce 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/app/mapreduce/LocalMRJobInfoFetcherTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/app/mapreduce/LocalMRJobInfoFetcherTest.java @@ -57,10 +57,10 @@ public static void beforeClass() throws Exception { public static Injector startMetricsService(CConfiguration conf) throws Exception { Injector injector = Guice.createInjector(new AppFabricTestModule(conf)); - injector.getInstance(TransactionManager.class).startAndWait(); + injector.getInstance(TransactionManager.class).startAsync().awaitRunning(); StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class)); - injector.getInstance(DatasetOpExecutorService.class).startAndWait(); - injector.getInstance(DatasetService.class).startAndWait(); + injector.getInstance(DatasetOpExecutorService.class).startAsync().awaitRunning(); + injector.getInstance(DatasetService.class).startAsync().awaitRunning(); return injector; } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/app/runtime/AbstractProgramRuntimeServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/app/runtime/AbstractProgramRuntimeServiceTest.java index 4b1922a33ca6..5772ef4ef8f0 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/app/runtime/AbstractProgramRuntimeServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/app/runtime/AbstractProgramRuntimeServiceTest.java @@ -110,7 +110,7 @@ public void testConcurrentStartLimit() throws Exception { Service service = new FastService(); ProgramController controller = new ProgramControllerServiceAdapter(service, programId.run(RunIds.generate())); - service.start(); + service.startAsync(); return controller; }; @@ -123,7 +123,7 @@ public void testConcurrentStartLimit() throws Exception { program, null, null); ProgramRuntimeService runtimeService = new TestProgramRuntimeService(cConf, runnerFactory, null, launchDispatcher); - runtimeService.startAndWait(); + runtimeService.startAsync().awaitRunning(); try { List controllers = new ArrayList<>(); for (int i = 0; i < 5; i++) { @@ -151,7 +151,7 @@ public void testConcurrentStartLimit() throws Exception { Assert.assertEquals(2, threadNames.size()); } finally { - runtimeService.stopAndWait(); + runtimeService.stopAsync().awaitTerminated(); } } @@ -166,7 +166,7 @@ public void testDeadlock() throws IOException, ExecutionException, InterruptedEx program, null, null); ProgramRuntimeService runtimeService = new TestProgramRuntimeService(cConf, runnerFactory, null, launchDispatcher); - runtimeService.startAndWait(); + runtimeService.startAsync().awaitRunning(); try { ProgramDescriptor descriptor = new ProgramDescriptor(program.getId(), null, NamespaceId.DEFAULT.artifact("test", "1.0")); @@ -178,7 +178,7 @@ public void testDeadlock() throws IOException, ExecutionException, InterruptedEx Tasks.waitFor(true, () -> runtimeService.list(ProgramType.WORKER).isEmpty(), 5, TimeUnit.SECONDS, 100, TimeUnit.MICROSECONDS); } finally { - runtimeService.stopAndWait(); + runtimeService.stopAsync().awaitTerminated(); } } @@ -190,20 +190,20 @@ public void testUpdateDeadLock() { ProgramId programId = NamespaceId.DEFAULT.app("dummyApp").program(ProgramType.WORKER, "dummy"); RunId runId = RunIds.generate(); ProgramRuntimeService.RuntimeInfo extraInfo = createRuntimeInfo(service, programId.run(runId)); - service.startAndWait(); + service.startAsync().awaitRunning(); ProgramRunnerFactory runnerFactory = createProgramRunnerFactory(); TestProgramRunDispatcher launchDispatcher = new TestProgramRunDispatcher(cConf, runnerFactory, null, null, null); TestProgramRuntimeService runtimeService = new TestProgramRuntimeService(cConf, runnerFactory, extraInfo, launchDispatcher); - runtimeService.startAndWait(); + runtimeService.startAsync().awaitRunning(); // The lookup will get deadlock for CDAP-3716 Assert.assertNotNull(runtimeService.lookup(programId, runId)); - service.stopAndWait(); + service.stopAsync().awaitTerminated(); - runtimeService.stopAndWait(); + runtimeService.stopAsync().awaitTerminated(); } @Test @@ -229,7 +229,7 @@ public ProgramLiveInfo getLiveInfo(ProgramId programId) { } }; - runtimeService.startAndWait(); + runtimeService.startAsync().awaitRunning(); try { try { ProgramDescriptor descriptor = new ProgramDescriptor(program.getId(), null, @@ -268,11 +268,11 @@ public ProgramLiveInfo getLiveInfo(ProgramId programId) { } } finally { - runtimeService.stopAndWait(); + runtimeService.stopAsync().awaitTerminated(); } } finally { - runtimeService.stopAndWait(); + runtimeService.stopAsync().awaitTerminated(); } } @@ -289,7 +289,7 @@ public void testTetheredRun() throws IOException, ExecutionException, Interrupte remoteClientFactory, true); ProgramRuntimeService runtimeService = new TestProgramRuntimeService(cConf, runnerFactory, null, launchDispatcher); - runtimeService.startAndWait(); + runtimeService.startAsync().awaitRunning(); try { ProgramDescriptor descriptor = new ProgramDescriptor(program.getId(), null, NamespaceId.DEFAULT.artifact("test", "1.0")); @@ -301,7 +301,7 @@ public void testTetheredRun() throws IOException, ExecutionException, Interrupte Tasks.waitFor(ProgramController.State.COMPLETED, controller::getState, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); } finally { - runtimeService.stopAndWait(); + runtimeService.stopAsync().awaitTerminated(); } } @@ -323,7 +323,7 @@ private ProgramRunnerFactory createProgramRunnerFactory(final Map startCompletion = service.start(); + service.startAsync(); controller.addListener(new AbstractListener() { private volatile boolean initCalled; @@ -86,8 +86,8 @@ public void alive() { } }, executor); - startCompletion.get(); - service.stopAndWait(); + service.awaitRunning(); + service.stopAsync().awaitTerminated(); } Assert.assertTrue(latch.await(5, TimeUnit.SECONDS)); diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/app/runtime/monitor/TrafficRelayServerTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/app/runtime/monitor/TrafficRelayServerTest.java index 9d7cefc64ac6..ac901ae39731 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/app/runtime/monitor/TrafficRelayServerTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/app/runtime/monitor/TrafficRelayServerTest.java @@ -45,7 +45,7 @@ public void testRelay() throws Exception { try { TrafficRelayServer relayServer = new TrafficRelayServer(InetAddress.getLoopbackAddress(), httpServer::getBindAddress); - relayServer.startAndWait(); + relayServer.startAsync().awaitRunning(); try { InetSocketAddress relayAddr = relayServer.getBindAddress(); @@ -63,7 +63,7 @@ public void testRelay() throws Exception { Assert.assertEquals("Testing", response.getResponseBodyAsString()); } finally { - relayServer.stopAndWait(); + relayServer.stopAsync().awaitTerminated(); } } finally { httpServer.stop(); diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/AppFabricTestHelper.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/AppFabricTestHelper.java index 778af9f0d967..07f6388a7090 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/AppFabricTestHelper.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/AppFabricTestHelper.java @@ -20,7 +20,7 @@ import com.google.common.base.Supplier; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import com.google.common.io.Closeables; + import com.google.common.util.concurrent.Service; import com.google.gson.Gson; import com.google.inject.AbstractModule; @@ -213,10 +213,18 @@ public static synchronized Injector getInjector(CConfiguration conf, @Nullable S * This must be called by all tests that create their injector through this class. */ public static void shutdown() { - Closeables.closeQuietly(metadataStorage); + try { + if (metadataStorage != null) { + metadataStorage.close(); + } + } catch (Exception e) { + // ignore + } if (services != null) { - Lists.reverse(services).forEach(Service::stopAndWait); + for (Service service : Lists.reverse(services)) { + service.stopAsync().awaitTerminated(); + } } InMemoryTableService.reset(); @@ -276,7 +284,7 @@ private static T startService(Injector injector, Class cls) { T instance = injector.getInstance(cls); if (instance instanceof Service) { services.add((Service) instance); - ((Service) instance).startAndWait(); + ((Service) instance).startAsync().awaitRunning(); } return instance; } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/deploy/pipeline/SystemMetadataWriterStageTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/deploy/pipeline/SystemMetadataWriterStageTest.java index bc3c67b31a24..8dc21fa39fb0 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/deploy/pipeline/SystemMetadataWriterStageTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/deploy/pipeline/SystemMetadataWriterStageTest.java @@ -77,12 +77,12 @@ public static void setup() { metadataStorage = injector.getInstance(MetadataStorage.class); metadataServiceClient = injector.getInstance(MetadataServiceClient.class); metadataSubscriber = injector.getInstance(MetadataSubscriberService.class); - metadataSubscriber.startAndWait(); + metadataSubscriber.startAsync().awaitRunning(); } @AfterClass public static void stop() { - metadataSubscriber.stopAndWait(); + metadataSubscriber.stopAsync().awaitTerminated(); AppFabricTestHelper.shutdown(); } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/namespace/StorageProviderNamespaceAdminTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/namespace/StorageProviderNamespaceAdminTest.java index 0384430654cf..9c7f653a2e1e 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/namespace/StorageProviderNamespaceAdminTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/namespace/StorageProviderNamespaceAdminTest.java @@ -76,12 +76,12 @@ protected void configure() { storageProviderNamespaceAdmin = injector.getInstance(StorageProviderNamespaceAdmin.class); // start the dataset service for namespace store to work transactionManager = injector.getInstance(TransactionManager.class); - transactionManager.startAndWait(); + transactionManager.startAsync().awaitRunning(); // Define all StructuredTable before starting any services that need StructuredTable StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class)); datasetService = injector.getInstance(DatasetService.class); - datasetService.startAndWait(); + datasetService.startAsync().awaitRunning(); // we don't use namespace admin here but the store because namespaceadmin will try to create the // home directory for namespace which we don't want. We just want to store the namespace meta in store // to look up during the delete. @@ -234,7 +234,7 @@ public void testLocalStorageProviderNamespaceAdminWithExistingTempDirectory() th @AfterClass public static void cleanup() throws Exception { - transactionManager.stopAndWait(); - datasetService.stopAndWait(); + transactionManager.stopAsync().awaitTerminated(); + datasetService.stopAsync().awaitTerminated(); } } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/preview/PreviewRunnerServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/preview/PreviewRunnerServiceTest.java index b0c7ffc66a59..67d6f05846d9 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/preview/PreviewRunnerServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/preview/PreviewRunnerServiceTest.java @@ -56,10 +56,10 @@ public void testStartAndStop() throws InterruptedException, ExecutionException, MockPreviewRunner mockRunner = new MockPreviewRunner(); MockPreviewRequestFetcher fetcher = new MockPreviewRequestFetcher(); PreviewRunnerService runnerService = new PreviewRunnerService(createCConf(), fetcher, mockRunner); - runnerService.startAndWait(); + runnerService.startAsync().awaitRunning(); Tasks.waitFor(true, () -> fetcher.fetchCount.get() > 0, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); - runnerService.stopAndWait(); + runnerService.stopAsync().awaitTerminated(); Tasks.waitFor(Service.State.TERMINATED, runnerService::state, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); } @@ -68,14 +68,14 @@ public void testStopPreview() throws InterruptedException, ExecutionException, T MockPreviewRunner mockRunner = new MockPreviewRunner(); MockPreviewRequestFetcher fetcher = new MockPreviewRequestFetcher(); PreviewRunnerService runnerService = new PreviewRunnerService(createCConf(), fetcher, mockRunner); - runnerService.startAndWait(); + runnerService.startAsync().awaitRunning(); ProgramId programId = NamespaceId.DEFAULT.app("app").program(ProgramType.WORKFLOW, "workflow"); fetcher.addRequest(new PreviewRequest(programId, null, null)); Tasks.waitFor(true, () -> mockRunner.requests.get(programId) != null, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); - runnerService.stopAndWait(); + runnerService.stopAsync().awaitTerminated(); Tasks.waitFor(PreviewStatus.Status.KILLED, () -> mockRunner.requests.get(programId).status.getStatus(), 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); Tasks.waitFor(Service.State.TERMINATED, runnerService::state, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); @@ -89,7 +89,7 @@ public void testMaxRuns() throws InterruptedException, ExecutionException, Timeo MockPreviewRunner mockRunner = new MockPreviewRunner(); MockPreviewRequestFetcher fetcher = new MockPreviewRequestFetcher(); PreviewRunnerService runnerService = new PreviewRunnerService(cConf, fetcher, mockRunner); - runnerService.startAndWait(); + runnerService.startAsync().awaitRunning(); ProgramId programId = NamespaceId.DEFAULT.app("app").program(ProgramType.WORKFLOW, "workflow"); fetcher.addRequest(new PreviewRequest(programId, null, null)); diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceProgramRunnerTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceProgramRunnerTest.java index d4ee727b2b5c..b2c077e03faf 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceProgramRunnerTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceProgramRunnerTest.java @@ -324,9 +324,11 @@ private void testMapreduceWithFile(String inputDatasetName, String inputPaths, Assert.assertFalse(resultLocation.isDirectory()); // read output and verify result - String line = CharStreams.readFirstLine( - CharStreams.newReaderSupplier( - Locations.newInputSupplier(resultLocation), Charsets.UTF_8)); + String line; + try (java.io.BufferedReader reader = new java.io.BufferedReader( + new java.io.InputStreamReader(resultLocation.getInputStream(), Charsets.UTF_8))) { + line = reader.readLine(); + } Assert.assertNotNull(line); String[] fields = line.split(outputSeparator == null ? ":" : outputSeparator); Assert.assertEquals(2, fields.length); diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceRunnerTestBase.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceRunnerTestBase.java index dada5c9f5f27..a17da83c0be9 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceRunnerTestBase.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceRunnerTestBase.java @@ -120,7 +120,7 @@ public static void beforeClass() throws Exception { NamespaceId.DEFAULT, DatasetDefinition.NO_ARGUMENTS, null, null); metricStore = injector.getInstance(MetricStore.class); - txService.startAndWait(); + txService.startAsync().awaitRunning(); // Always create the default namespace injector.getInstance(NamespaceAdmin.class).create(NamespaceMeta.DEFAULT); @@ -128,7 +128,7 @@ public static void beforeClass() throws Exception { @AfterClass public static void afterClass() { - txService.stopAndWait(); + txService.stopAsync().awaitTerminated(); AppFabricTestHelper.shutdown(); } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/batch/dataset/input/MapReduceWithMultipleInputsTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/batch/dataset/input/MapReduceWithMultipleInputsTest.java index a28a67b501fe..ec747ac7517e 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/batch/dataset/input/MapReduceWithMultipleInputsTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/batch/dataset/input/MapReduceWithMultipleInputsTest.java @@ -80,8 +80,11 @@ public void testSimpleJoin() throws Exception { // will only be 1 part file, due to the small amount of data Location outputLocation = outputFileSet.getBaseLocation().append("output").append("part-r-00000"); - List lines = CharStreams.readLines( - CharStreams.newReaderSupplier(Locations.newInputSupplier(outputLocation), Charsets.UTF_8)); + List lines; + try (java.io.BufferedReader reader = new java.io.BufferedReader( + new java.io.InputStreamReader(outputLocation.getInputStream(), Charsets.UTF_8))) { + lines = CharStreams.readLines(reader); + } Assert.assertEquals(ImmutableList.of("1 Bob 75", "2 Samuel 18", "3 Joe 60"), lines); diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/batch/dataset/output/MapReduceWithMultipleOutputsTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/batch/dataset/output/MapReduceWithMultipleOutputsTest.java index 74c9f8983ce7..72becff8460a 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/batch/dataset/output/MapReduceWithMultipleOutputsTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/batch/dataset/output/MapReduceWithMultipleOutputsTest.java @@ -70,7 +70,10 @@ public void testMultipleOutputs() throws Exception { private List readFromOutput(FileSet fileSet, String relativePath) throws IOException { // small amount of data, so expect all data from just 1 file Location location = fileSet.getLocation(relativePath).append("part-m-00000"); - return CharStreams.readLines(CharStreams.newReaderSupplier(Locations.newInputSupplier(location), Charsets.UTF_8)); + try (java.io.BufferedReader reader = new java.io.BufferedReader( + new java.io.InputStreamReader(location.getInputStream(), Charsets.UTF_8))) { + return CharStreams.readLines(reader); + } } @Test diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionJobMainTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionJobMainTest.java index 7ba50a84eba7..04e1696391df 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionJobMainTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionJobMainTest.java @@ -48,11 +48,11 @@ public void testJobEnvironment() throws Exception { Map properties = jobEnv.getProperties(); ZKClientService zkClient = ZKClientService.Builder.of(properties.get(Constants.Zookeeper.QUORUM)).build(); - zkClient.startAndWait(); + zkClient.startAsync().awaitRunning(); try { Assert.assertNotNull(zkClient.exists("/").get()); } finally { - zkClient.stopAndWait(); + zkClient.stopAsync().awaitTerminated(); } } finally { runner.destroy(); diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/monitor/InternalServiceRoutingHandlerTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/monitor/InternalServiceRoutingHandlerTest.java index 867110e5c8c1..7363973dafbc 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/monitor/InternalServiceRoutingHandlerTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/monitor/InternalServiceRoutingHandlerTest.java @@ -102,7 +102,7 @@ protected void configure() { ); internalRouterService = injector.getInstance(InternalRouterService.class); - internalRouterService.startAndWait(); + internalRouterService.startAsync().awaitRunning(); mockService = NettyHttpService.builder(MOCK_SERVICE) .setHost(InetAddress.getLocalHost().getCanonicalHostName()) @@ -118,7 +118,7 @@ protected void configure() { public void afterTest() throws Exception { mockServiceCancellable.cancel(); mockService.stop(); - internalRouterService.stopAndWait(); + internalRouterService.stopAsync().awaitTerminated(); } @Test diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/monitor/RuntimeClientServerTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/monitor/RuntimeClientServerTest.java index 6c848de4a609..b26983c8e96f 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/monitor/RuntimeClientServerTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/monitor/RuntimeClientServerTest.java @@ -146,12 +146,12 @@ protected void configure() { messagingService = injector.getInstance(MessagingService.class); if (messagingService instanceof Service) { - ((Service) messagingService).startAndWait(); + ((Service) messagingService).startAsync().awaitRunning(); } messagingService.createTopic(new DefaultTopicMetadata(NamespaceId.SYSTEM.topic("topic"))); runtimeServer = injector.getInstance(RuntimeServer.class); - runtimeServer.startAndWait(); + runtimeServer.startAsync().awaitRunning(); runtimeClient = injector.getInstance(RuntimeClient.class); locationFactory = injector.getInstance(LocationFactory.class); @@ -160,9 +160,9 @@ protected void configure() { @After public void afterTest() { logEntries.clear(); - runtimeServer.stopAndWait(); + runtimeServer.stopAsync().awaitTerminated(); if (messagingService instanceof Service) { - ((Service) messagingService).stopAndWait(); + ((Service) messagingService).stopAsync().awaitTerminated(); } } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/monitor/RuntimeClientServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/monitor/RuntimeClientServiceTest.java index a01ed0700847..5cada649de7b 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/monitor/RuntimeClientServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/monitor/RuntimeClientServiceTest.java @@ -198,11 +198,11 @@ protected void configure() { messagingService = injector.getInstance(MessagingService.class); if (messagingService instanceof Service) { - ((Service) messagingService).startAndWait(); + ((Service) messagingService).startAsync().awaitRunning(); } runtimeServer = injector.getInstance(RuntimeServer.class); - runtimeServer.startAndWait(); + runtimeServer.startAsync().awaitRunning(); // Injector for the client side clientCConf = CConfiguration.create(); @@ -244,7 +244,7 @@ protected void configure() { clientMessagingService = clientInjector.getInstance(MessagingService.class); if (clientMessagingService instanceof Service) { - ((Service) clientMessagingService).startAndWait(); + ((Service) clientMessagingService).startAsync().awaitRunning(); } clientProgramStatePublisher = clientInjector.getInstance( ProgramStatePublisher.class); @@ -253,16 +253,16 @@ protected void configure() { @After public void afterTest() { if (runtimeClientService != null) { - runtimeClientService.stopAndWait(); + runtimeClientService.stopAsync().awaitTerminated(); } runtimeClientService = null; if (clientMessagingService instanceof Service) { - ((Service) clientMessagingService).stopAndWait(); + ((Service) clientMessagingService).stopAsync().awaitTerminated(); } - runtimeServer.stopAndWait(); + runtimeServer.stopAsync().awaitTerminated(); if (messagingService instanceof Service) { - ((Service) messagingService).stopAndWait(); + ((Service) messagingService).stopAsync().awaitTerminated(); } } @@ -270,7 +270,7 @@ public void afterTest() { public void testBasicRelay() throws Exception { runtimeClientService = clientInjector.getInstance( RuntimeClientService.class); - runtimeClientService.startAndWait(); + runtimeClientService.startAsync().awaitRunning(); // Send some messages to multiple topics in the client side TMS, they should get replicated to the server side TMS. MessagingContext messagingContext = new MultiThreadMessagingContext( clientMessagingService); @@ -327,7 +327,7 @@ public void testRelayWithAggregation() throws Exception { runtimeClientService = clientInjector.getInstance( RuntimeClientService.class); - runtimeClientService.startAndWait(); + runtimeClientService.startAsync().awaitRunning(); Map tags = new HashMap<>(); tags.put("key1", "value1"); tags.put("key2", "value2"); @@ -435,7 +435,7 @@ public void testRelayWithAggregation() throws Exception { public void testProgramTerminate() throws Exception { runtimeClientService = clientInjector.getInstance( RuntimeClientService.class); - runtimeClientService.startAndWait(); + runtimeClientService.startAsync().awaitRunning(); MessagingContext messagingContext = new MultiThreadMessagingContext( clientMessagingService); MessagePublisher messagePublisher = messagingContext.getDirectMessagePublisher(); @@ -483,11 +483,23 @@ public void testProgramTerminate() throws Exception { public void testRuntimeClientStop() throws Exception { runtimeClientService = clientInjector.getInstance( RuntimeClientService.class); - runtimeClientService.startAndWait(); + runtimeClientService.startAsync().awaitRunning(); ProgramStateWriter programStateWriter = new MessagingProgramStateWriter( clientProgramStatePublisher); - ListenableFuture stopFuture = runtimeClientService.stop(); + com.google.common.util.concurrent.SettableFuture stopFuture = + com.google.common.util.concurrent.SettableFuture.create(); + runtimeClientService.addListener(new Service.Listener() { + @Override + public void terminated(Service.State from) { + stopFuture.set(Service.State.TERMINATED); + } + @Override + public void failed(Service.State from, Throwable failure) { + stopFuture.setException(failure); + } + }, com.google.common.util.concurrent.MoreExecutors.directExecutor()); + runtimeClientService.stopAsync(); try { stopFuture.get(2, TimeUnit.SECONDS); Assert.fail("Expected runtime client service not stopped"); @@ -507,7 +519,7 @@ public void testRuntimeClientStop() throws Exception { public void testExternalStop() throws Exception { runtimeClientService = clientInjector.getInstance( RuntimeClientService.class); - runtimeClientService.startAndWait(); + runtimeClientService.startAsync().awaitRunning(); ProgramStateWriter programStateWriter = new MessagingProgramStateWriter( clientProgramStatePublisher); MessagingContext messagingContext = new MultiThreadMessagingContext( @@ -527,7 +539,19 @@ public void testExternalStop() throws Exception { messagePublisher.publish(NamespaceId.SYSTEM.getNamespace(), topic, "msg1" + topic, "msg2" + topic); - ListenableFuture stopFuture = runtimeClientService.stop(); + com.google.common.util.concurrent.SettableFuture stopFuture = + com.google.common.util.concurrent.SettableFuture.create(); + runtimeClientService.addListener(new Service.Listener() { + @Override + public void terminated(Service.State from) { + stopFuture.set(Service.State.TERMINATED); + } + @Override + public void failed(Service.State from, Throwable failure) { + stopFuture.setException(failure); + } + }, com.google.common.util.concurrent.MoreExecutors.directExecutor()); + runtimeClientService.stopAsync(); try { stopFuture.get(2, TimeUnit.SECONDS); Assert.fail("Expected runtime client service not stopped"); diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/monitor/RuntimeServiceRoutingTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/monitor/RuntimeServiceRoutingTest.java index 1ef6ca49a8f0..201a79dd5554 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/monitor/RuntimeServiceRoutingTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/monitor/RuntimeServiceRoutingTest.java @@ -117,7 +117,7 @@ protected void bindRequestValidator() { bind(RuntimeRequestValidator.class).toInstance((programRunId, request) -> { String authHeader = request.headers().get(HttpHeaderNames.AUTHORIZATION); String expected = "Bearer " + Base64.getEncoder().encodeToString( - Hashing.md5().hashString(programRunId.toString()).asBytes()); + Hashing.md5().hashString(programRunId.toString(), StandardCharsets.UTF_8).asBytes()); if (!expected.equals(authHeader)) { throw new UnauthenticatedException("Program run " + programRunId + " is not authorized"); } @@ -143,12 +143,12 @@ protected void configure() { messagingService = injector.getInstance(MessagingService.class); if (messagingService instanceof Service) { - ((Service) messagingService).startAndWait(); + ((Service) messagingService).startAsync().awaitRunning(); } messagingService.createTopic(new DefaultTopicMetadata(NamespaceId.SYSTEM.topic("topic"))); runtimeServer = injector.getInstance(RuntimeServer.class); - runtimeServer.startAndWait(); + runtimeServer.startAsync().awaitRunning(); mockService = NettyHttpService.builder(MOCK_SERVICE) .setHost(InetAddress.getLocalHost().getCanonicalHostName()) @@ -164,9 +164,9 @@ protected void configure() { public void afterTest() throws Exception { mockServiceCancellable.cancel(); mockService.stop(); - runtimeServer.stopAndWait(); + runtimeServer.stopAsync().awaitTerminated(); if (messagingService instanceof Service) { - ((Service) messagingService).stopAndWait(); + ((Service) messagingService).stopAsync().awaitTerminated(); } } @@ -269,8 +269,8 @@ public String getName() { @Override public Credential getCredentials() { - String credentialValue = Base64.getEncoder().encodeToString(Hashing.md5().hashString(programRunId.toString()) - .asBytes()); + String credentialValue = Base64.getEncoder().encodeToString( + Hashing.md5().hashString(programRunId.toString(), StandardCharsets.UTF_8).asBytes()); return new Credential(credentialValue, Credential.CredentialType.EXTERNAL_BEARER); } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/monitor/proxy/ServiceSocksProxyTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/monitor/proxy/ServiceSocksProxyTest.java index 110b9cab2186..22abacc85cae 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/monitor/proxy/ServiceSocksProxyTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/monitor/proxy/ServiceSocksProxyTest.java @@ -81,7 +81,7 @@ public static void init() throws Exception { discoveryService.register(ResolvingDiscoverable.of((new Discoverable("test-service", httpService.getBindAddress())))); proxyServer = new ServiceSocksProxy(discoveryService, (user, pass) -> USER.equals(user) && PASS.equals(pass)); - proxyServer.startAndWait(); + proxyServer.startAsync().awaitRunning(); defaultProxySelector = ProxySelector.getDefault(); @@ -113,7 +113,7 @@ protected PasswordAuthentication getPasswordAuthentication() { public static void finish() throws Exception { Authenticator.setDefault(null); ProxySelector.setDefault(defaultProxySelector); - proxyServer.stopAndWait(); + proxyServer.stopAsync().awaitTerminated(); httpService.stop(); } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/schedule/queue/NoSqlJobQueueTableTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/schedule/queue/NoSqlJobQueueTableTest.java index 65cd9f225d2c..2d350dcd9aab 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/schedule/queue/NoSqlJobQueueTableTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/schedule/queue/NoSqlJobQueueTableTest.java @@ -61,7 +61,7 @@ public static void beforeClass() throws IOException, TableAlreadyExistsException cConf.set(Constants.Dataset.DATA_STORAGE_IMPLEMENTATION, Constants.Dataset.DATA_STORAGE_NOSQL); txManager = new TransactionManager(new Configuration()); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); Injector injector = Guice.createInjector( new ConfigModule(cConf), @@ -90,7 +90,7 @@ protected void configure() { @AfterClass public static void afterClass() { - txManager.stopAndWait(); + txManager.stopAsync().awaitTerminated(); } @Override diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/schedule/store/DatasetBasedTimeScheduleStoreTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/schedule/store/DatasetBasedTimeScheduleStoreTest.java index 40460053d486..5b5d40313db2 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/schedule/store/DatasetBasedTimeScheduleStoreTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/schedule/store/DatasetBasedTimeScheduleStoreTest.java @@ -124,20 +124,20 @@ protected void configure() { } }); txService = injector.getInstance(TransactionManager.class); - txService.startAndWait(); + txService.startAsync().awaitRunning(); StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class)); dsOpsService = injector.getInstance(DatasetOpExecutorService.class); - dsOpsService.startAndWait(); + dsOpsService.startAsync().awaitRunning(); dsService = injector.getInstance(DatasetService.class); - dsService.startAndWait(); + dsService.startAsync().awaitRunning(); transactionRunner = injector.getInstance(TransactionRunner.class); } @AfterClass public static void afterClass() { - dsService.stopAndWait(); - dsOpsService.stopAndWait(); - txService.stopAndWait(); + dsService.stopAsync().awaitTerminated(); + dsOpsService.stopAsync().awaitTerminated(); + txService.stopAsync().awaitTerminated(); } private static void schedulerSetup(boolean enablePersistence) throws SchedulerException { diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/service/http/HttpHandlerGeneratorTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/service/http/HttpHandlerGeneratorTest.java index 3efcdab0b02f..4f36f6c2be03 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/service/http/HttpHandlerGeneratorTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/service/http/HttpHandlerGeneratorTest.java @@ -275,7 +275,11 @@ public void onFinish(HttpServiceResponder responder) throws Exception { @Override public void onError(HttpServiceResponder responder, Throwable failureCause) { validateTransaction(); - Closeables.closeQuietly(channel); + try { + channel.close(); + } catch (IOException e) { + // ignore + } LOG.error("Failed when handling upload", failureCause); } @@ -481,7 +485,9 @@ protected FileHandler createHandler() { String.format("http://%s:%d/content/download/test.txt", bindAddress.getHostName(), bindAddress.getPort())).openConnection(); try { - ByteStreams.copy(urlConn.getInputStream(), Files.newOutputStreamSupplier(downloadFile)); + try (FileOutputStream fos = new FileOutputStream(downloadFile)) { + ByteStreams.copy(urlConn.getInputStream(), fos); + } } finally { urlConn.disconnect(); } @@ -506,7 +512,9 @@ protected FileHandler createHandler() { urlConn.setDoOutput(true); urlConn.setRequestMethod("POST"); Files.copy(file, urlConn.getOutputStream()); - ByteStreams.copy(urlConn.getInputStream(), Files.newOutputStreamSupplier(downloadFile)); + try (FileOutputStream fos = new FileOutputStream(downloadFile)) { + ByteStreams.copy(urlConn.getInputStream(), fos); + } Assert.assertEquals(200, urlConn.getResponseCode()); Assert.assertTrue(Files.equal(file, downloadFile)); } finally { @@ -558,8 +566,7 @@ protected NoAnnotationHandler createHandler() { bindAddress.getHostName(), bindAddress.getPort())).openConnection(); urlConn.setReadTimeout(2000); urlConn.setDoOutput(true); - ByteStreams.copy(ByteStreams.newInputStreamSupplier("Hello".getBytes(Charsets.UTF_8)), - urlConn.getOutputStream()); + urlConn.getOutputStream().write("Hello".getBytes(Charsets.UTF_8)); Assert.assertEquals("Hello test", new String(ByteStreams.toByteArray(urlConn.getInputStream()), Charsets.UTF_8)); diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/worker/WorkerProgramRunnerTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/worker/WorkerProgramRunnerTest.java index 66e8204661e9..e4c369096fd4 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/worker/WorkerProgramRunnerTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/worker/WorkerProgramRunnerTest.java @@ -119,12 +119,12 @@ public static void beforeClass() { NamespaceId.DEFAULT, DatasetDefinition.NO_ARGUMENTS, null, null); metricStore = injector.getInstance(MetricStore.class); - txService.startAndWait(); + txService.startAsync().awaitRunning(); } @AfterClass public static void afterClass() { - txService.stopAndWait(); + txService.stopAsync().awaitTerminated(); AppFabricTestHelper.shutdown(); } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/AppFabricProcessorServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/AppFabricProcessorServiceTest.java index a147f710fff3..c18779e3ad0a 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/AppFabricProcessorServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/AppFabricProcessorServiceTest.java @@ -29,10 +29,12 @@ public void startStopService() { try { Injector injector = AppFabricTestHelper.getInjector(); AppFabricProcessorService service = injector.getInstance(AppFabricProcessorService.class); - Service.State state = service.startAndWait(); + service.startAsync().awaitRunning(); + Service.State state = service.state(); Assert.assertSame(state, Service.State.RUNNING); - state = service.stopAndWait(); + service.stopAsync().awaitTerminated(); + state = service.state(); Assert.assertSame(state, Service.State.TERMINATED); } finally { AppFabricTestHelper.shutdown(); diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/AppFabricServerTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/AppFabricServerTest.java index 6a1ca2f3cb01..e12d21fb1171 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/AppFabricServerTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/AppFabricServerTest.java @@ -49,14 +49,16 @@ public void startStopServer() throws Exception { try { AppFabricServer server = injector.getInstance(AppFabricServer.class); DiscoveryServiceClient discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class); - Service.State state = server.startAndWait(); + server.startAsync().awaitRunning(); + Service.State state = server.state(); Assert.assertSame(state, Service.State.RUNNING); final EndpointStrategy endpointStrategy = new RandomEndpointStrategy( () -> discoveryServiceClient.discover(Constants.Service.APP_FABRIC_HTTP)); Assert.assertNotNull(endpointStrategy.pick(5, TimeUnit.SECONDS)); - state = server.stopAndWait(); + server.stopAsync().awaitTerminated(); + state = server.state(); Assert.assertSame(state, Service.State.TERMINATED); Tasks.waitFor(true, () -> endpointStrategy.pick() == null, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); @@ -74,7 +76,7 @@ public void testSsl() throws IOException { try { final DiscoveryServiceClient discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class); AppFabricServer appFabricServer = injector.getInstance(AppFabricServer.class); - appFabricServer.startAndWait(); + appFabricServer.startAsync().awaitRunning(); Assert.assertTrue(appFabricServer.isRunning()); Supplier endpointStrategySupplier = Suppliers.memoize( @@ -91,7 +93,7 @@ public void testSsl() throws IOException { // Would throw exception if the server does not support ssl. // "javax.net.ssl.SSLException: Unrecognized SSL message, plaintext connection?" socket.startHandshake(); - appFabricServer.stopAndWait(); + appFabricServer.stopAsync().awaitTerminated(); } finally { AppFabricTestHelper.shutdown(); } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/DefaultSecureStoreServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/DefaultSecureStoreServiceTest.java index a8b3ae336149..e2ddf1e8024d 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/DefaultSecureStoreServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/DefaultSecureStoreServiceTest.java @@ -93,9 +93,9 @@ public static void setup() throws Exception { final Injector injector = AppFabricTestHelper.getInjector(cConf, sConf); discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class); appFabricServer = injector.getInstance(AppFabricServer.class); - appFabricServer.startAndWait(); + appFabricServer.startAsync().awaitRunning(); appFabricProcessor = injector.getInstance(AppFabricProcessorService.class); - appFabricProcessor.startAndWait(); + appFabricProcessor.startAsync().awaitRunning(); waitForService(Constants.Service.DATASET_MANAGER); secureStore = injector.getInstance(SecureStore.class); secureStoreManager = injector.getInstance(SecureStoreManager.class); @@ -127,8 +127,8 @@ private static void waitForService(String service) { @AfterClass public static void cleanup() { - appFabricServer.stopAndWait(); - appFabricProcessor.stopAndWait(); + appFabricServer.stopAsync().awaitTerminated(); + appFabricProcessor.stopAsync().awaitTerminated(); AppFabricTestHelper.shutdown(); } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ProgramLifecycleServiceAuthorizationTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ProgramLifecycleServiceAuthorizationTest.java index 5f785986d2ae..d84d0d993a45 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ProgramLifecycleServiceAuthorizationTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ProgramLifecycleServiceAuthorizationTest.java @@ -78,9 +78,9 @@ public static void setup() throws Exception { final Injector injector = AppFabricTestHelper.getInjector(cConf); permissionManager = injector.getInstance(PermissionManager.class); appFabricServer = injector.getInstance(AppFabricServer.class); - appFabricServer.startAndWait(); + appFabricServer.startAsync().awaitRunning(); appFabricProcessor = injector.getInstance(AppFabricProcessorService.class); - appFabricProcessor.startAndWait(); + appFabricProcessor.startAsync().awaitRunning(); programLifecycleService = injector.getInstance(ProgramLifecycleService.class); // Wait for the default namespace creation @@ -161,8 +161,8 @@ public void testProgramList() throws Exception { @AfterClass public static void tearDown() { - appFabricServer.stopAndWait(); - appFabricProcessor.stopAndWait(); + appFabricServer.stopAsync().awaitTerminated(); + appFabricProcessor.stopAsync().awaitTerminated(); AppFabricTestHelper.shutdown(); } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ProgramLifecycleServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ProgramLifecycleServiceTest.java index 12f5700ff401..35393fa01ee5 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ProgramLifecycleServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ProgramLifecycleServiceTest.java @@ -77,12 +77,12 @@ public static void beforeClass() throws Throwable { programLifecycleService = injector.getInstance(ProgramLifecycleService.class); profileService = injector.getInstance(ProfileService.class); provisioningService = injector.getInstance(ProvisioningService.class); - provisioningService.startAndWait(); + provisioningService.startAsync().awaitRunning(); } @AfterClass public static void shutdown() { - provisioningService.stopAndWait(); + provisioningService.stopAsync().awaitTerminated(); } @Test diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ProgramRunStatusMonitorServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ProgramRunStatusMonitorServiceTest.java index cf0bebb6d857..408c8eb081a5 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ProgramRunStatusMonitorServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/ProgramRunStatusMonitorServiceTest.java @@ -125,7 +125,7 @@ public RuntimeInfo lookup(ProgramId programId, RunId runId) { = new ProgramRunStatusMonitorService(cConf, store, testService, metricsCollectionService, new NoOpProgramStateWriter(), 5, 3, 2, 2); - programRunStatusMonitorService.startAndWait(); + programRunStatusMonitorService.startAsync().awaitRunning(); Assert.assertEquals(1, latch.getCount()); programRunStatusMonitorService.terminatePrograms(); Assert.assertTrue(latch.await(10, TimeUnit.SECONDS)); @@ -198,7 +198,7 @@ public RuntimeInfo lookup(ProgramId programId, RunId runId) { = new ProgramRunStatusMonitorService(cConf, store, testService, metricsCollectionService, new NoOpProgramStateWriter(), 5, 3, 2, 2); - programRunStatusMonitorService.startAndWait(); + programRunStatusMonitorService.startAsync().awaitRunning(); Assert.assertEquals(1, latch.getCount()); programRunStatusMonitorService.terminatePrograms(); Assert.assertTrue(latch.await(10, TimeUnit.SECONDS)); @@ -238,7 +238,7 @@ public void testStoppingRemoteTetheredProgramsBeyondTerminateTimeAreKilled() thr ProgramRunStatusMonitorService programRunStatusMonitorService = new ProgramRunStatusMonitorService(cConf, store, testService, metricsCollectionService, psw, 5, 3, 2, 2); - programRunStatusMonitorService.startAndWait(); + programRunStatusMonitorService.startAsync().awaitRunning(); Assert.assertEquals(1, latch.getCount()); programRunStatusMonitorService.terminatePrograms(); Assert.assertTrue(latch.await(10, TimeUnit.SECONDS)); diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/SystemProgramManagementServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/SystemProgramManagementServiceTest.java index 620e3225942f..9601e8aa6ab7 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/SystemProgramManagementServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/SystemProgramManagementServiceTest.java @@ -73,7 +73,7 @@ public static void setup() { progmMgmtSvc = new SystemProgramManagementService(getInjector().getInstance(CConfiguration.class), getInjector().getInstance(ProgramRuntimeService.class), programLifecycleService); - progmMgmtSvc.stopAndWait(); + progmMgmtSvc.stopAsync().awaitTerminated(); } @AfterClass diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/AppFabricTestBase.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/AppFabricTestBase.java index e74c4b10b691..8ca905a5fc0f 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/AppFabricTestBase.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/AppFabricTestBase.java @@ -20,8 +20,6 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; -import com.google.common.io.Closeables; -import com.google.common.io.InputSupplier; import com.google.common.util.concurrent.Service; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -67,6 +65,7 @@ import io.cdap.cdap.common.internal.remote.RemoteClientFactory; import io.cdap.cdap.common.io.CaseInsensitiveEnumTypeAdapterFactory; import io.cdap.cdap.common.io.Locations; +import io.cdap.cdap.common.lang.ThrowingSupplier; import io.cdap.cdap.common.service.Retries; import io.cdap.cdap.common.test.AppJarHelper; import io.cdap.cdap.common.test.PluginJarHelper; @@ -268,38 +267,38 @@ protected static void initializeAndStartServices(CConfiguration cConf, Module ov messagingService = injector.getInstance(MessagingService.class); if (messagingService instanceof Service) { - ((Service) messagingService).startAndWait(); + ((Service) messagingService).startAsync().awaitRunning(); } txManager = injector.getInstance(TransactionManager.class); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); // Define all StructuredTable before starting any services that need StructuredTable StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class)); metadataStorage = injector.getInstance(MetadataStorage.class); metadataStorage.createIndex(); dsOpService = injector.getInstance(DatasetOpExecutorService.class); - dsOpService.startAndWait(); + dsOpService.startAsync().awaitRunning(); datasetService = injector.getInstance(DatasetService.class); - datasetService.startAndWait(); + datasetService.startAsync().awaitRunning(); appFabricServer = injector.getInstance(AppFabricServer.class); - appFabricServer.startAndWait(); + appFabricServer.startAsync().awaitRunning(); appFabricProcessorService = injector.getInstance(AppFabricProcessorService.class); - appFabricProcessorService.startAndWait(); + appFabricProcessorService.startAsync().awaitRunning(); DiscoveryServiceClient discoveryClient = injector.getInstance(DiscoveryServiceClient.class); appFabricEndpointStrategy = new RandomEndpointStrategy( () -> discoveryClient.discover(Constants.Service.APP_FABRIC_HTTP)); txClient = injector.getInstance(TransactionSystemClient.class); metricsCollectionService = injector.getInstance(MetricsCollectionService.class); - metricsCollectionService.startAndWait(); + metricsCollectionService.startAsync().awaitRunning(); serviceStore = injector.getInstance(ServiceStore.class); - serviceStore.startAndWait(); + serviceStore.startAsync().awaitRunning(); metadataService = injector.getInstance(MetadataService.class); - metadataService.startAndWait(); + metadataService.startAsync().awaitRunning(); metadataSubscriberService = injector.getInstance(MetadataSubscriberService.class); - metadataSubscriberService.startAndWait(); + metadataSubscriberService.startAsync().awaitRunning(); logQueryService = injector.getInstance(LogQueryService.class); - logQueryService.startAndWait(); + logQueryService.startAsync().awaitRunning(); locationFactory = getInjector().getInstance(LocationFactory.class); datasetClient = new DatasetClient(getClientConfig(discoveryClient, Constants.Service.DATASET_MANAGER)); remoteClientFactory = new RemoteClientFactory(discoveryClient, @@ -323,20 +322,26 @@ protected static void initializeAndStartServices(CConfiguration cConf, Module ov @AfterClass public static void afterClass() { - appFabricServer.stopAndWait(); - appFabricProcessorService.stopAndWait(); - metricsCollectionService.stopAndWait(); - datasetService.stopAndWait(); - dsOpService.stopAndWait(); - txManager.stopAndWait(); - serviceStore.stopAndWait(); - metadataSubscriberService.stopAndWait(); - metadataService.stopAndWait(); - logQueryService.stopAndWait(); + appFabricServer.stopAsync().awaitTerminated(); + appFabricProcessorService.stopAsync().awaitTerminated(); + metricsCollectionService.stopAsync().awaitTerminated(); + datasetService.stopAsync().awaitTerminated(); + dsOpService.stopAsync().awaitTerminated(); + txManager.stopAsync().awaitTerminated(); + serviceStore.stopAsync().awaitTerminated(); + metadataSubscriberService.stopAsync().awaitTerminated(); + metadataService.stopAsync().awaitTerminated(); + logQueryService.stopAsync().awaitTerminated(); if (messagingService instanceof Service) { - ((Service) messagingService).stopAndWait(); + ((Service) messagingService).stopAsync().awaitTerminated(); + } + try { + if (metadataStorage != null) { + metadataStorage.close(); + } + } catch (Exception e) { + // ignore } - Closeables.closeQuietly(metadataStorage); } protected static CConfiguration createBasicCconf() throws IOException { @@ -502,7 +507,8 @@ protected HttpResponse addPluginArtifact(Id.Artifact artifactId, Class cls, } // add an artifact and return the response code - protected HttpResponse addArtifact(Id.Artifact artifactId, InputSupplier artifactContents, + protected HttpResponse addArtifact(Id.Artifact artifactId, + ThrowingSupplier artifactContents, Set parents) throws Exception { return addArtifact(artifactId, artifactContents, parents, null); } @@ -510,7 +516,8 @@ protected HttpResponse addArtifact(Id.Artifact artifactId, InputSupplier artifactContents, + private HttpResponse addArtifact(Id.Artifact artifactId, + ThrowingSupplier artifactContents, Set parents, @Nullable String pluginClassesJson) throws Exception { String path = getVersionedApiPath("artifacts/" + artifactId.getName(), artifactId.getNamespace().getId()); @@ -527,7 +534,12 @@ private HttpResponse addArtifact(Id.Artifact artifactId, InputSupplier() { + @Override + public InputStream getInput() throws IOException { + return artifactContents.get(); + } + }); return HttpRequests.execute(builder.build(), httpRequestConfig); } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/remote/RemoteNamespaceQueryTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/remote/RemoteNamespaceQueryTest.java index ae94d92c25c5..1fe12ca5e1ed 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/remote/RemoteNamespaceQueryTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/remote/RemoteNamespaceQueryTest.java @@ -63,11 +63,11 @@ public static void setup() throws Exception { cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMPORARY_FOLDER.newFolder().getAbsolutePath()); Injector injector = AppFabricTestHelper.getInjector(cConf); txManager = injector.getInstance(TransactionManager.class); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); datasetService = injector.getInstance(DatasetService.class); - datasetService.startAndWait(); + datasetService.startAsync().awaitRunning(); appFabricServer = injector.getInstance(AppFabricServer.class); - appFabricServer.startAndWait(); + appFabricServer.startAsync().awaitRunning(); DiscoveryServiceClient discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class); waitForService(discoveryServiceClient, Constants.Service.DATASET_MANAGER); waitForService(discoveryServiceClient, Constants.Service.APP_FABRIC_HTTP); @@ -78,9 +78,9 @@ public static void setup() throws Exception { @AfterClass public static void tearDown() { - appFabricServer.stopAndWait(); - datasetService.stopAndWait(); - txManager.stopAndWait(); + appFabricServer.stopAsync().awaitTerminated(); + datasetService.stopAsync().awaitTerminated(); + txManager.stopAsync().awaitTerminated(); AppFabricTestHelper.shutdown(); } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/remote/RemotePermissionsTestBase.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/remote/RemotePermissionsTestBase.java index 595e4f0a42ca..4aa03ed521eb 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/remote/RemotePermissionsTestBase.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/remote/RemotePermissionsTestBase.java @@ -87,7 +87,7 @@ protected static void setup() throws IOException, InterruptedException { Injector injector = AppFabricTestHelper.getInjector(cConf); discoveryService = injector.getInstance(DiscoveryServiceClient.class); appFabricServer = injector.getInstance(AppFabricServer.class); - appFabricServer.startAndWait(); + appFabricServer.startAsync().awaitRunning(); waitForService(Constants.Service.APP_FABRIC_HTTP); accessEnforcer = injector.getInstance(RemoteAccessEnforcer.class); permissionManager = injector.getInstance(PermissionManager.class); @@ -226,7 +226,7 @@ private void assertUnauthorized(Retries.Runnable runnable) thro @AfterClass public static void tearDown() { - appFabricServer.stopAndWait(); + appFabricServer.stopAsync().awaitTerminated(); AppFabricTestHelper.shutdown(); } } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/state/AppStateHandlerTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/state/AppStateHandlerTest.java index 3ca2f4ef46ec..9d60556e1384 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/state/AppStateHandlerTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/state/AppStateHandlerTest.java @@ -88,7 +88,7 @@ public static void setup() throws Exception { applicationLifecycleService = injector.getInstance(ApplicationLifecycleService.class); txManager = injector.getInstance(TransactionManager.class); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); // Endpoint for all state APIs endpoint = "namespaces/" + NAMESPACE_1 + "/apps/" + APP_NAME + "/states/" + STATE_KEY; @@ -104,7 +104,7 @@ public static void teardown() throws Exception { } if (txManager != null) { - txManager.stopAndWait(); + txManager.stopAsync().awaitTerminated(); } } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/state/AppStateTableTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/state/AppStateTableTest.java index a41ce9acb37e..43a8c4ac2ceb 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/state/AppStateTableTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/store/state/AppStateTableTest.java @@ -66,7 +66,7 @@ public static void setup() throws Exception { Injector injector = getInjector(); txManager = injector.getInstance(TransactionManager.class); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); transactionRunner = getInjector().getInstance(TransactionRunner.class); @@ -79,7 +79,7 @@ public static void setup() throws Exception { @AfterClass public static void teardown() throws Exception { if (txManager != null) { - txManager.stopAndWait(); + txManager.stopAsync().awaitTerminated(); } } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/worker/TaskWorkerMetricsTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/worker/TaskWorkerMetricsTest.java index 4533afa424b4..fa0715918400 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/worker/TaskWorkerMetricsTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/worker/TaskWorkerMetricsTest.java @@ -84,7 +84,7 @@ protected void publish(Iterator metrics) { } }; - mockMetricsCollector.startAndWait(); + mockMetricsCollector.startAsync().awaitRunning(); InMemoryDiscoveryService discoveryService = new InMemoryDiscoveryService(); AeadCipher aeadCipher = new NoOpAeadCipher(); taskWorkerService = new TaskWorkerService(cConf, sConf, discoveryService, discoveryService, @@ -93,7 +93,7 @@ protected void publish(Iterator metrics) { auditLogContexts -> {}, aeadCipher)); taskWorkerStateFuture = TaskWorkerTestUtil.getServiceCompletionFuture(taskWorkerService); // start the service - taskWorkerService.startAndWait(); + taskWorkerService.startAsync().awaitRunning(); InetSocketAddress addr = taskWorkerService.getBindAddress(); this.uri = URI.create(String.format("http://%s:%s", addr.getHostName(), addr.getPort())); } @@ -101,7 +101,7 @@ protected void publish(Iterator metrics) { @After public void afterTest() { if (taskWorkerService != null) { - taskWorkerService.stopAndWait(); + taskWorkerService.stopAsync().awaitTerminated(); } } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/worker/TaskWorkerServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/worker/TaskWorkerServiceTest.java index 6fd7227670b2..484039e4daab 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/worker/TaskWorkerServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/worker/TaskWorkerServiceTest.java @@ -105,7 +105,7 @@ public void beforeTest() throws Exception { serviceCompletionFuture = TaskWorkerTestUtil.getServiceCompletionFuture( taskWorkerService); // start the service - taskWorkerService.startAndWait(); + taskWorkerService.startAsync().awaitRunning(); this.taskWorkerService = taskWorkerService; securityManager = System.getSecurityManager(); System.setSecurityManager(new NoExitSecurityManager()); @@ -114,7 +114,7 @@ public void beforeTest() throws Exception { @After public void afterTest() { if (taskWorkerService != null) { - taskWorkerService.stopAndWait(); + taskWorkerService.stopAsync().awaitTerminated(); taskWorkerService = null; } System.setSecurityManager(securityManager); @@ -135,7 +135,7 @@ public void testPeriodicRestart() { serviceCompletionFuture = TaskWorkerTestUtil.getServiceCompletionFuture( taskWorkerService); // start the service - taskWorkerService.startAndWait(); + taskWorkerService.startAsync().awaitRunning(); TaskWorkerTestUtil.waitForServiceCompletion(serviceCompletionFuture); Assert.assertEquals(Service.State.TERMINATED, taskWorkerService.state()); @@ -156,7 +156,7 @@ public void testPeriodicRestartWithInflightRequest() throws IOException { serviceCompletionFuture = TaskWorkerTestUtil.getServiceCompletionFuture( taskWorkerService); // start the service - taskWorkerService.startAndWait(); + taskWorkerService.startAsync().awaitRunning(); InetSocketAddress addr = taskWorkerService.getBindAddress(); URI uri = URI.create( @@ -199,7 +199,7 @@ public void testPeriodicRestartWithNeverEndingInflightRequest() { serviceCompletionFuture = TaskWorkerTestUtil.getServiceCompletionFuture( taskWorkerService); // start the service - taskWorkerService.startAndWait(); + taskWorkerService.startAsync().awaitRunning(); new Thread( () -> { @@ -244,7 +244,7 @@ public void testRestartAfterMultipleExecutions() throws IOException { serviceCompletionFuture = TaskWorkerTestUtil.getServiceCompletionFuture( taskWorkerService); // start the service - taskWorkerService.startAndWait(); + taskWorkerService.startAsync().awaitRunning(); InetSocketAddress addr = taskWorkerService.getBindAddress(); URI uri = URI.create( @@ -370,7 +370,7 @@ public void testConcurrentRequestsWithIsolationDisabled() throws Exception { createSConf(), discoveryService, discoveryService, metricsCollectionService, new CommonNettyHttpServiceFactory(cConf, metricsCollectionService, auditLogContexts -> {}, aeadCipher)); - taskWorkerService.startAndWait(); + taskWorkerService.startAsync().awaitRunning(); InetSocketAddress addr = taskWorkerService.getBindAddress(); URI uri = URI.create( String.format("http://%s:%s", addr.getHostName(), addr.getPort())); @@ -413,7 +413,7 @@ public void testConcurrentRequestsWithIsolationDisabled() throws Exception { } catch (TimeoutException e) { // ignore. } - taskWorkerService.stopAndWait(); + taskWorkerService.stopAsync().awaitTerminated(); Assert.assertEquals(2, okResponse); Assert.assertEquals(concurrentRequests, okResponse + conflictResponse); Assert.assertEquals(Service.State.TERMINATED, taskWorkerService.state()); @@ -433,7 +433,7 @@ public void testRestartWithConcurrentRequests() throws Exception { new CommonNettyHttpServiceFactory(cConf, metricsCollectionService, auditLogContexts -> {}, aeadCipher)); serviceCompletionFuture = TaskWorkerTestUtil.getServiceCompletionFuture( taskWorkerService); - taskWorkerService.startAndWait(); + taskWorkerService.startAsync().awaitRunning(); InetSocketAddress addr = taskWorkerService.getBindAddress(); URI uri = URI.create( String.format("http://%s:%s", addr.getHostName(), addr.getPort())); diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/worker/TaskWorkerTestUtil.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/worker/TaskWorkerTestUtil.java index d2992bc53235..1ca1535adfb8 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/worker/TaskWorkerTestUtil.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/worker/TaskWorkerTestUtil.java @@ -20,7 +20,7 @@ import com.google.common.util.concurrent.Uninterruptibles; import java.util.concurrent.CompletableFuture; import org.apache.twill.common.Threads; -import org.apache.twill.internal.ServiceListenerAdapter; + /** * Common TaskWorker test utility functions @@ -38,7 +38,7 @@ private TaskWorkerTestUtil() { */ static CompletableFuture getServiceCompletionFuture(TaskWorkerService taskWorker) { CompletableFuture future = new CompletableFuture<>(); - taskWorker.addListener(new ServiceListenerAdapter() { + taskWorker.addListener(new Service.Listener() { @Override public void terminated(Service.State from) { future.complete(from); diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/worker/sidecar/ArtifactLocalizerServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/worker/sidecar/ArtifactLocalizerServiceTest.java index 16254b402aa7..7f343175ed9e 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/worker/sidecar/ArtifactLocalizerServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/worker/sidecar/ArtifactLocalizerServiceTest.java @@ -90,7 +90,7 @@ cConf, new ArtifactLocalizer(cConf, remoteClientFactory, (namespaceId, retryStra new NoOpAeadCipher()), remoteClientFactory, new NoOpRemoteAuthenticator()); // start the service - artifactLocalizerService.startAndWait(); + artifactLocalizerService.startAsync().awaitRunning(); return artifactLocalizerService; } @@ -104,7 +104,7 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { - this.localizerService.stopAndWait(); + this.localizerService.stopAsync().awaitTerminated(); } @Test diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/worker/system/SystemWorkerServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/worker/system/SystemWorkerServiceTest.java index d29dff65625b..22854ab8b4e3 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/worker/system/SystemWorkerServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/worker/system/SystemWorkerServiceTest.java @@ -116,14 +116,14 @@ public void beforeTest() throws IOException { new RunnableTaskModule(discoveryService, discoveryService, new NoOpMetricsCollectionService())), new AuthenticationTestContext(), new NoOpAccessController()); - service.startAndWait(); + service.startAsync().awaitRunning(); this.systemWorkerService = service; } @After public void afterTest() { if (systemWorkerService != null) { - systemWorkerService.stopAndWait(); + systemWorkerService.stopAsync().awaitTerminated(); systemWorkerService = null; } } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/audit/AuditPublishTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/audit/AuditPublishTest.java index 9c93e434f4d6..fe049309a491 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/audit/AuditPublishTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/audit/AuditPublishTest.java @@ -87,7 +87,7 @@ public static void init() throws Exception { Injector injector = AppFabricTestHelper.getInjector(cConf, new AuditModule()); messagingService = injector.getInstance(MessagingService.class); if (messagingService instanceof Service) { - ((Service) messagingService).startAndWait(); + ((Service) messagingService).startAsync().awaitRunning(); } auditTopic = NamespaceId.SYSTEM.topic(cConf.get(Constants.Audit.TOPIC)); } @@ -95,7 +95,7 @@ public static void init() throws Exception { @AfterClass public static void stop() { if (messagingService instanceof Service) { - ((Service) messagingService).stopAndWait(); + ((Service) messagingService).stopAsync().awaitTerminated(); } AppFabricTestHelper.shutdown(); } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/capability/CapabilityManagementServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/capability/CapabilityManagementServiceTest.java index 2f24fb2d850a..8706babe9bf6 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/capability/CapabilityManagementServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/capability/CapabilityManagementServiceTest.java @@ -100,7 +100,7 @@ public static void setup() { programLifecycleService = getInjector().getInstance(ProgramLifecycleService.class); programStateWriter = getInjector().getInstance(ProgramStateWriter.class); runtimeService = getInjector().getInstance(ProgramRuntimeService.class); - capabilityManagementService.stopAndWait(); + capabilityManagementService.stopAsync().awaitTerminated(); } @AfterClass diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialProviderTestBase.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialProviderTestBase.java index fc6082f7ee34..f0b3c0cbb222 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialProviderTestBase.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialProviderTestBase.java @@ -94,7 +94,7 @@ protected void configure() { } }); txManager = injector.getInstance(TransactionManager.class); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); contextAccessEnforcer = injector.getInstance(ContextAccessEnforcer.class); CredentialProviderStore.create(injector .getInstance(StructuredTableAdmin.class)); @@ -131,7 +131,7 @@ protected void configure() { @AfterClass public static void teardown() throws Exception { if (txManager != null) { - txManager.stopAndWait(); + txManager.stopAsync().awaitTerminated(); } } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/events/ProgramStatusEventPublisherMetricsTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/events/ProgramStatusEventPublisherMetricsTest.java index 8428ff36dea6..52a9629200ab 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/events/ProgramStatusEventPublisherMetricsTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/events/ProgramStatusEventPublisherMetricsTest.java @@ -51,13 +51,13 @@ public void testMetrics() throws IOException { MetricsProvider mockMetricsProvider = getMockMetricsProvider(); List metricValuesList = new ArrayList<>(); MetricsCollectionService mockMetricsCollectionService = getMockCollectionService(metricValuesList); - mockMetricsCollectionService.startAndWait(); + mockMetricsCollectionService.startAsync().awaitRunning(); ProgramStatusEventPublisher programStatusEventPublisher = new ProgramStatusEventPublisher( CConfiguration.create(), null, mockMetricsCollectionService, null, mockMetricsProvider ); programStatusEventPublisher.initialize(Collections.singleton(new DummyEventWriter())); programStatusEventPublisher.processMessages(null, getMockNotification()); - mockMetricsCollectionService.stopAndWait(); + mockMetricsCollectionService.stopAsync().awaitTerminated(); Assert.assertSame(1, metricValuesList.size()); Assert.assertTrue(containsMetric(metricValuesList.get(0), Constants.Metrics.ProgramEvent.PUBLISHED_COUNT)); } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/operation/OperationLifecycleManagerTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/operation/OperationLifecycleManagerTest.java index 6a6efdd61ff8..809b8f5625b6 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/operation/OperationLifecycleManagerTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/operation/OperationLifecycleManagerTest.java @@ -16,7 +16,7 @@ package io.cdap.cdap.internal.operation; -import com.google.common.io.Closeables; + import com.google.inject.AbstractModule; import com.google.inject.Guice; import com.google.inject.Injector; @@ -98,7 +98,11 @@ protected void configure() { @AfterClass public static void afterClass() { - Closeables.closeQuietly(postgres); + try { + postgres.close(); + } catch (java.io.IOException e) { + // ignore + } } @Test diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/operation/OperationNotificationSingleTopicSubscriberServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/operation/OperationNotificationSingleTopicSubscriberServiceTest.java index 1b7498eb9fe5..4696ca7ee42a 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/operation/OperationNotificationSingleTopicSubscriberServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/operation/OperationNotificationSingleTopicSubscriberServiceTest.java @@ -19,7 +19,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.google.common.io.Closeables; + import com.google.gson.Gson; import com.google.inject.AbstractModule; import com.google.inject.Guice; @@ -99,7 +99,11 @@ protected void configure() { @AfterClass public static void afterClass() { - Closeables.closeQuietly(pg); + try { + pg.close(); + } catch (IOException e) { + // ignore + } } @Test diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/operation/SqlOperationRunsStoreTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/operation/SqlOperationRunsStoreTest.java index 25b5df7aea83..9008d929117f 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/operation/SqlOperationRunsStoreTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/operation/SqlOperationRunsStoreTest.java @@ -16,7 +16,7 @@ package io.cdap.cdap.internal.operation; -import com.google.common.io.Closeables; + import com.google.inject.AbstractModule; import com.google.inject.Guice; import com.google.inject.Injector; @@ -71,6 +71,10 @@ protected void configure() { @AfterClass public static void afterClass() { - Closeables.closeQuietly(pg); + try { + pg.close(); + } catch (IOException e) { + // ignore + } } } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/profile/ProfileMetadataTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/profile/ProfileMetadataTest.java index 2e118e21b7ab..9f6ef15869ce 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/profile/ProfileMetadataTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/profile/ProfileMetadataTest.java @@ -48,12 +48,12 @@ public class ProfileMetadataTest extends AppFabricTestBase { @BeforeClass public static void setUp() { metadataSubscriberService = getInjector().getInstance(MetadataSubscriberService.class); - metadataSubscriberService.startAndWait(); + metadataSubscriberService.startAsync().awaitRunning(); } @AfterClass public static void tearDown() { - metadataSubscriberService.stopAndWait(); + metadataSubscriberService.stopAsync().awaitTerminated(); } @Test diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/provision/ProvisioningServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/provision/ProvisioningServiceTest.java index 1aeec499b62c..57505143ae88 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/provision/ProvisioningServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/provision/ProvisioningServiceTest.java @@ -98,32 +98,32 @@ public static void setupClass() throws Exception { Injector injector = Guice.createInjector(new AppFabricTestModule(cConf)); txManager = injector.getInstance(TransactionManager.class); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); // Define all StructuredTable before starting any services that need StructuredTable StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class)); datasetService = injector.getInstance(DatasetService.class); - datasetService.startAndWait(); + datasetService.startAsync().awaitRunning(); messagingService = injector.getInstance(MessagingService.class); provisionerStore = injector.getInstance(ProvisionerStore.class); if (messagingService instanceof Service) { - ((Service) messagingService).startAndWait(); + ((Service) messagingService).startAsync().awaitRunning(); } provisioningService = injector.getInstance(ProvisioningService.class); - provisioningService.startAndWait(); + provisioningService.startAsync().awaitRunning(); transactionRunner = injector.getInstance(TransactionRunner.class); } @AfterClass public static void cleanupClass() { - provisioningService.stopAndWait(); - datasetService.stopAndWait(); - txManager.stopAndWait(); + provisioningService.stopAsync().awaitTerminated(); + datasetService.stopAsync().awaitTerminated(); + txManager.stopAsync().awaitTerminated(); if (messagingService instanceof Service) { - ((Service) messagingService).stopAndWait(); + ((Service) messagingService).stopAsync().awaitTerminated(); } } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/tethering/ArtifactCacheServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/tethering/ArtifactCacheServiceTest.java index 61b55bae62f9..a8a16a35e277 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/tethering/ArtifactCacheServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/tethering/ArtifactCacheServiceTest.java @@ -82,7 +82,7 @@ public void setUp() throws Exception { cConf, artifactCache, tetheringStore, null, discoveryService, new CommonNettyHttpServiceFactory(cConf, new NoOpMetricsCollectionService(), auditLogContexts -> {}, new NoOpAeadCipher())); - artifactCacheService.startAndWait(); + artifactCacheService.startAsync().awaitRunning(); getInjector().getInstance(ArtifactRepository.class).clear(NamespaceId.DEFAULT); LocationFactory locationFactory = getInjector().getInstance(LocationFactory.class); appJar = AppJarHelper.createDeploymentJar(locationFactory, TaskWorkerServiceTest.TestRunnableClass.class); @@ -97,7 +97,7 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { - artifactCacheService.stopAndWait(); + artifactCacheService.stopAsync().awaitTerminated(); artifactRepository.deleteArtifact(artifactId); appJar.delete(); deletePeer(); diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/tethering/TetheringClientHandlerTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/tethering/TetheringClientHandlerTest.java index 07cc8edee887..bb77d435e885 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/tethering/TetheringClientHandlerTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/tethering/TetheringClientHandlerTest.java @@ -189,7 +189,7 @@ protected void configure() { }); tetheringStore = new TetheringStore(injector.getInstance(TransactionRunner.class)); txManager = injector.getInstance(TransactionManager.class); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); profileService = injector.getInstance(ProfileService.class); namespaceAdmin = injector.getInstance(NamespaceAdmin.class); namespaceAdmin.create(new NamespaceMeta.Builder().setName(NAMESPACE_1).build()); @@ -203,7 +203,7 @@ public static void teardown() throws Exception { namespaceAdmin.delete(new NamespaceId(NAMESPACE_2)); namespaceAdmin.delete(new NamespaceId(NAMESPACE_3)); if (txManager != null) { - txManager.stopAndWait(); + txManager.stopAsync().awaitTerminated(); } } @@ -259,7 +259,8 @@ public void setUp() throws Exception { tetheringEventPublisher = new TetheringProgramEventPublisher(cConf, tetheringStore, messagingService, injector.getInstance(ProgramRunRecordFetcher.class), transactionRunner); - Assert.assertEquals(Service.State.RUNNING, tetheringEventPublisher.startAndWait()); + tetheringEventPublisher.startAsync().awaitRunning(); + Assert.assertEquals(Service.State.RUNNING, tetheringEventPublisher.state()); tetheringAgentService = new TetheringAgentService(cConf, tetheringStore, injector.getInstance(ProgramStateWriter.class), @@ -268,14 +269,17 @@ public void setUp() throws Exception { injector.getInstance(LocationFactory.class), injector.getInstance(ProvisionerNotifier.class), injector.getInstance(NamespaceQueryAdmin.class)); - Assert.assertEquals(Service.State.RUNNING, tetheringAgentService.startAndWait()); + tetheringAgentService.startAsync().awaitRunning(); + Assert.assertEquals(Service.State.RUNNING, tetheringAgentService.state()); } @After public void tearDown() throws Exception { deleteTetheringIfNeeded(SERVER_INSTANCE); - Assert.assertEquals(Service.State.TERMINATED, tetheringEventPublisher.stopAndWait()); - Assert.assertEquals(Service.State.TERMINATED, tetheringAgentService.stopAndWait()); + tetheringEventPublisher.stopAsync().awaitTerminated(); + Assert.assertEquals(Service.State.TERMINATED, tetheringEventPublisher.state()); + tetheringAgentService.stopAsync().awaitTerminated(); + Assert.assertEquals(Service.State.TERMINATED, tetheringAgentService.state()); } @Test diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/tethering/TetheringServerHandlerTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/tethering/TetheringServerHandlerTest.java index be6b386bccc1..69b2ce818bd0 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/tethering/TetheringServerHandlerTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/tethering/TetheringServerHandlerTest.java @@ -174,21 +174,21 @@ protected void configure() { tetheringStore = new TetheringStore(injector.getInstance(TransactionRunner.class)); messagingService = injector.getInstance(MessagingService.class); if (messagingService instanceof Service) { - ((Service) messagingService).startAndWait(); + ((Service) messagingService).startAsync().awaitRunning(); } messagingProgramStatePublisher = injector.getInstance(MessagingProgramStatePublisher.class); profileService = injector.getInstance(ProfileService.class); txManager = injector.getInstance(TransactionManager.class); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); } @AfterClass public static void teardown() { if (txManager != null) { - txManager.stopAndWait(); + txManager.stopAsync().awaitTerminated(); } if (messagingService instanceof Service) { - ((Service) messagingService).stopAndWait(); + ((Service) messagingService).stopAsync().awaitTerminated(); } } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/tethering/runtime/spi/runtimejob/TetheringRuntimeJobManagerTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/tethering/runtime/spi/runtimejob/TetheringRuntimeJobManagerTest.java index 52ec53b9932a..d6faee812ecf 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/tethering/runtime/spi/runtimejob/TetheringRuntimeJobManagerTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/tethering/runtime/spi/runtimejob/TetheringRuntimeJobManagerTest.java @@ -137,11 +137,11 @@ protected void configure() { StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class)); messagingService = injector.getInstance(MessagingService.class); if (messagingService instanceof Service) { - ((Service) messagingService).startAndWait(); + ((Service) messagingService).startAsync().awaitRunning(); } txManager = injector.getInstance(TransactionManager.class); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); tetheringStore = injector.getInstance(TetheringStore.class); PeerMetadata metadata = new PeerMetadata(Collections.singletonList(new NamespaceAllocation(TETHERED_NAMESPACE_NAME, null, @@ -163,11 +163,11 @@ protected void configure() { @AfterClass public static void tearDown() throws TopicNotFoundException, IOException { if (txManager != null) { - txManager.stopAndWait(); + txManager.stopAsync().awaitTerminated(); } messagingService.deleteTopic(topicId); if (messagingService instanceof Service) { - ((Service) messagingService).stopAndWait(); + ((Service) messagingService).stopAsync().awaitTerminated(); } } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/metadata/MetadataAdminAuthorizationTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/metadata/MetadataAdminAuthorizationTest.java index f0037a1175a9..cddb1f3ba851 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/metadata/MetadataAdminAuthorizationTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/metadata/MetadataAdminAuthorizationTest.java @@ -83,9 +83,9 @@ public static void setup() throws Exception { permissionManager = injector.getInstance(PermissionManager.class); appFabricServer = injector.getInstance(AppFabricServer.class); - appFabricServer.startAndWait(); + appFabricServer.startAsync().awaitRunning(); appFabricProcessor = injector.getInstance(AppFabricProcessorService.class); - appFabricProcessor.startAndWait(); + appFabricProcessor.startAsync().awaitRunning(); // Wait for the default namespace creation String user = AuthorizationUtil.getEffectiveMasterUser(cConf); @@ -173,8 +173,8 @@ public void testSearch() throws Exception { @AfterClass public static void tearDown() { - appFabricServer.stopAndWait(); - appFabricProcessor.stopAndWait(); + appFabricServer.stopAsync().awaitTerminated(); + appFabricProcessor.stopAsync().awaitTerminated(); AppFabricTestHelper.shutdown(); } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/metadata/SystemMetadataAuditPublishTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/metadata/SystemMetadataAuditPublishTest.java index 7d871f6b36fc..ff675a126d81 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/metadata/SystemMetadataAuditPublishTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/metadata/SystemMetadataAuditPublishTest.java @@ -70,14 +70,14 @@ protected void configure() { namespaceAdmin = injector.getInstance(NamespaceAdmin.class); scheduler = injector.getInstance(Scheduler.class); if (scheduler instanceof Service) { - ((Service) scheduler).startAndWait(); + ((Service) scheduler).startAsync().awaitRunning(); } } @AfterClass public static void tearDown() { if (scheduler instanceof Service) { - ((Service) scheduler).stopAndWait(); + ((Service) scheduler).stopAsync().awaitTerminated(); } AppFabricTestHelper.shutdown(); } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/scheduler/CoreSchedulerServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/scheduler/CoreSchedulerServiceTest.java index 9095855c80f4..84661daa83b4 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/scheduler/CoreSchedulerServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/scheduler/CoreSchedulerServiceTest.java @@ -146,7 +146,7 @@ public static void setup() { cConf = getInjector().getInstance(CConfiguration.class); scheduler = getInjector().getInstance(Scheduler.class); if (scheduler instanceof Service) { - ((Service) scheduler).startAndWait(); + ((Service) scheduler).startAsync().awaitRunning(); } messagingService = getInjector().getInstance(MessagingService.class); store = getInjector().getInstance(Store.class); @@ -156,7 +156,7 @@ public static void setup() { @AfterClass public static void tearDown() { if (scheduler instanceof Service) { - ((Service) scheduler).stopAndWait(); + ((Service) scheduler).stopAsync().awaitTerminated(); } } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/security/auth/AuditLogSingleTopicSubscriberServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/security/auth/AuditLogSingleTopicSubscriberServiceTest.java index 48d6d3e12335..065d9f0b774b 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/security/auth/AuditLogSingleTopicSubscriberServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/security/auth/AuditLogSingleTopicSubscriberServiceTest.java @@ -17,7 +17,7 @@ package io.cdap.cdap.security.auth; import com.google.common.collect.ImmutableList; -import com.google.common.io.Closeables; + import com.google.inject.AbstractModule; import com.google.inject.Guice; import com.google.inject.Injector; @@ -94,7 +94,11 @@ protected void configure() { @AfterClass public static void afterClass() { - Closeables.closeQuietly(pg); + try { + pg.close(); + } catch (IOException e) { + // ignore + } } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/security/impersonation/DefaultUGIProviderTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/security/impersonation/DefaultUGIProviderTest.java index 6f9ced673f1c..29b46c16e7cb 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/security/impersonation/DefaultUGIProviderTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/security/impersonation/DefaultUGIProviderTest.java @@ -236,7 +236,9 @@ private void verifyCaching(DefaultUGIProvider provider, ImpersonationRequest ali private Location copyFileToHDFS(Location hdfsKeytabDir, File localFile) throws IOException { Location remoteFile = hdfsKeytabDir.append(localFile.getName()); Assert.assertTrue(remoteFile.createNew()); - Files.copy(localFile, Locations.newOutputSupplier(remoteFile)); + try (java.io.OutputStream out = remoteFile.getOutputStream()) { + Files.copy(localFile, out); + } return remoteFile; } diff --git a/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/main/java/io/cdap/cdap/datapipeline/DataPipelineApp.java b/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/main/java/io/cdap/cdap/datapipeline/DataPipelineApp.java index 1023c65b9911..ed55713d82ba 100644 --- a/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/main/java/io/cdap/cdap/datapipeline/DataPipelineApp.java +++ b/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/main/java/io/cdap/cdap/datapipeline/DataPipelineApp.java @@ -16,7 +16,7 @@ package io.cdap.cdap.datapipeline; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableSet; import io.cdap.cdap.api.app.AbstractApplication; import io.cdap.cdap.api.app.ApplicationUpdateContext; @@ -64,7 +64,7 @@ public void configure() { return; } - setDescription(Objects.firstNonNull(config.getDescription(), DEFAULT_DESCRIPTION)); + setDescription(MoreObjects.firstNonNull(config.getDescription(), DEFAULT_DESCRIPTION)); addWorkflow(new SmartWorkflow(config, supportedPluginTypes, getConfigurer())); String timeSchedule = config.getSchedule(); diff --git a/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/test/java/io/cdap/cdap/datapipeline/preview/PreviewDataPipelineTest.java b/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/test/java/io/cdap/cdap/datapipeline/preview/PreviewDataPipelineTest.java index b8459bb632b2..6e056f3aa6a6 100644 --- a/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/test/java/io/cdap/cdap/datapipeline/preview/PreviewDataPipelineTest.java +++ b/cdap-app-templates/cdap-etl/cdap-data-pipeline-base/src/test/java/io/cdap/cdap/datapipeline/preview/PreviewDataPipelineTest.java @@ -673,8 +673,8 @@ private void validateMetric(long expected, ApplicationId previewId, // Min sleep time is 10ms, max sleep time is 1 seconds long sleepMillis = TimeUnit.SECONDS.toMillis(1); - Stopwatch stopwatch = new Stopwatch().start(); - while (value < expected && stopwatch.elapsedTime(TimeUnit.SECONDS) < 20) { + Stopwatch stopwatch = Stopwatch.createStarted(); + while (value < expected && stopwatch.elapsed(TimeUnit.SECONDS) < 20) { TimeUnit.MILLISECONDS.sleep(sleepMillis); value = getTotalMetric(tags, metricName, previewManager); } diff --git a/cdap-app-templates/cdap-etl/cdap-data-pipeline3_2.12/pom.xml b/cdap-app-templates/cdap-etl/cdap-data-pipeline3_2.12/pom.xml index 651f2c4c1e59..862abaf589ff 100644 --- a/cdap-app-templates/cdap-etl/cdap-data-pipeline3_2.12/pom.xml +++ b/cdap-app-templates/cdap-etl/cdap-data-pipeline3_2.12/pom.xml @@ -66,6 +66,12 @@ ${project.version} provided + + io.cdap.cdap + cdap-common + ${project.version} + provided + io.cdap.cdap cdap-system-app-api @@ -197,6 +203,15 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + net.alchim31.maven scala-maven-plugin diff --git a/cdap-app-templates/cdap-etl/cdap-data-streams-base/src/main/java/io/cdap/cdap/datastreams/DataStreamsApp.java b/cdap-app-templates/cdap-etl/cdap-data-streams-base/src/main/java/io/cdap/cdap/datastreams/DataStreamsApp.java index 5e75c36d7239..9a971e8116af 100644 --- a/cdap-app-templates/cdap-etl/cdap-data-streams-base/src/main/java/io/cdap/cdap/datastreams/DataStreamsApp.java +++ b/cdap-app-templates/cdap-etl/cdap-data-streams-base/src/main/java/io/cdap/cdap/datastreams/DataStreamsApp.java @@ -16,7 +16,7 @@ package io.cdap.cdap.datastreams; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableSet; import io.cdap.cdap.api.app.AbstractApplication; import io.cdap.cdap.api.app.ApplicationUpdateContext; @@ -36,7 +36,7 @@ public class DataStreamsApp extends AbstractApplication { @Override public void configure() { DataStreamsConfig config = getConfig(); - setDescription(Objects.firstNonNull(config.getDescription(), "Data Streams Application")); + setDescription(MoreObjects.firstNonNull(config.getDescription(), "Data Streams Application")); DataStreamsPipelineSpec spec; try { diff --git a/cdap-app-templates/cdap-etl/cdap-data-streams-base/src/test/java/io/cdap/cdap/datastreams/preview/PreviewDataStreamsTest.java b/cdap-app-templates/cdap-etl/cdap-data-streams-base/src/test/java/io/cdap/cdap/datastreams/preview/PreviewDataStreamsTest.java index bc264400752a..1ddb6f60afc5 100644 --- a/cdap-app-templates/cdap-etl/cdap-data-streams-base/src/test/java/io/cdap/cdap/datastreams/preview/PreviewDataStreamsTest.java +++ b/cdap-app-templates/cdap-etl/cdap-data-streams-base/src/test/java/io/cdap/cdap/datastreams/preview/PreviewDataStreamsTest.java @@ -178,8 +178,8 @@ private void validateMetric(long expected, ApplicationId previewId, // Min sleep time is 10ms, max sleep time is 1 seconds long sleepMillis = TimeUnit.SECONDS.toMillis(1); - Stopwatch stopwatch = new Stopwatch().start(); - while (value < expected && stopwatch.elapsedTime(TimeUnit.SECONDS) < 5) { + Stopwatch stopwatch = Stopwatch.createStarted(); + while (value < expected && stopwatch.elapsed(TimeUnit.SECONDS) < 5) { TimeUnit.MILLISECONDS.sleep(sleepMillis); value = getTotalMetric(tags, metricName, previewManager); } diff --git a/cdap-app-templates/cdap-etl/cdap-data-streams3_2.12/pom.xml b/cdap-app-templates/cdap-etl/cdap-data-streams3_2.12/pom.xml index e95bde1b5d24..55567fdf18a8 100644 --- a/cdap-app-templates/cdap-etl/cdap-data-streams3_2.12/pom.xml +++ b/cdap-app-templates/cdap-etl/cdap-data-streams3_2.12/pom.xml @@ -153,6 +153,15 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + org.apache.maven.plugins diff --git a/cdap-app-templates/cdap-etl/cdap-etl-api-spark/pom.xml b/cdap-app-templates/cdap-etl/cdap-etl-api-spark/pom.xml index c683ff6c0cfb..8c7c48196c5e 100644 --- a/cdap-app-templates/cdap-etl/cdap-etl-api-spark/pom.xml +++ b/cdap-app-templates/cdap-etl/cdap-etl-api-spark/pom.xml @@ -67,6 +67,15 @@ scala-maven-plugin 3.3.1 + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + diff --git a/cdap-app-templates/cdap-etl/cdap-etl-api/pom.xml b/cdap-app-templates/cdap-etl/cdap-etl-api/pom.xml index 3315643ef3b2..b6fa94b24c0f 100644 --- a/cdap-app-templates/cdap-etl/cdap-etl-api/pom.xml +++ b/cdap-app-templates/cdap-etl/cdap-etl-api/pom.xml @@ -43,4 +43,18 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 17 + 17 + + + + + diff --git a/cdap-app-templates/cdap-etl/cdap-etl-batch/pom.xml b/cdap-app-templates/cdap-etl/cdap-etl-batch/pom.xml index e9c300b786be..950c3463a090 100644 --- a/cdap-app-templates/cdap-etl/cdap-etl-batch/pom.xml +++ b/cdap-app-templates/cdap-etl/cdap-etl-batch/pom.xml @@ -141,6 +141,15 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + net.alchim31.maven scala-maven-plugin diff --git a/cdap-app-templates/cdap-etl/cdap-etl-core/pom.xml b/cdap-app-templates/cdap-etl/cdap-etl-core/pom.xml index 32df33d4b7ae..186756edc16c 100644 --- a/cdap-app-templates/cdap-etl/cdap-etl-core/pom.xml +++ b/cdap-app-templates/cdap-etl/cdap-etl-core/pom.xml @@ -79,5 +79,16 @@ + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 17 + 17 + + + diff --git a/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/AbstractServiceRetryableMacroEvaluator.java b/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/AbstractServiceRetryableMacroEvaluator.java index 684d62972945..4d6265985403 100644 --- a/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/AbstractServiceRetryableMacroEvaluator.java +++ b/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/AbstractServiceRetryableMacroEvaluator.java @@ -80,11 +80,11 @@ public String evaluate(String macroFunction, String... args) throws InvalidMacro RETRY_DELAY_MULTIPLIER - RETRY_DELAY_MULTIPLIER * RETRY_RANDOMIZE_FACTOR; double maxMultiplier = RETRY_DELAY_MULTIPLIER + RETRY_DELAY_MULTIPLIER * RETRY_RANDOMIZE_FACTOR; - Stopwatch stopWatch = new Stopwatch().start(); + Stopwatch stopWatch = Stopwatch.createStarted(); int retryCount = 0; RetryableException retryableException = null; try { - while (stopWatch.elapsedTime(TimeUnit.MILLISECONDS) < TIMEOUT_MILLIS) { + while (stopWatch.elapsed(TimeUnit.MILLISECONDS) < TIMEOUT_MILLIS) { try { retryCount++; return evaluateMacro(macroFunction, args); @@ -150,9 +150,9 @@ public Map evaluateMap(String macroFunction, String... args) double minMultiplier = RETRY_DELAY_MULTIPLIER - RETRY_DELAY_MULTIPLIER * RETRY_RANDOMIZE_FACTOR; double maxMultiplier = RETRY_DELAY_MULTIPLIER + RETRY_DELAY_MULTIPLIER * RETRY_RANDOMIZE_FACTOR; Exception ex = null; - Stopwatch stopWatch = new Stopwatch().start(); + Stopwatch stopWatch = Stopwatch.createStarted(); try { - while (stopWatch.elapsedTime(TimeUnit.MILLISECONDS) < TIMEOUT_MILLIS) { + while (stopWatch.elapsed(TimeUnit.MILLISECONDS) < TIMEOUT_MILLIS) { try { return evaluateMacroMap(macroFunction, args); } catch (RetryableException e) { diff --git a/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/PipelineTypeValidator.java b/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/PipelineTypeValidator.java index e7b2d352ac35..3eea2f08fbc5 100644 --- a/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/PipelineTypeValidator.java +++ b/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/PipelineTypeValidator.java @@ -90,7 +90,7 @@ static void validateTypes(List unresTypeList) { Type firstType = resTypeList.get(i); Type secondType = resTypeList.get(i + 1); // Check if secondType can accept firstType - Preconditions.checkArgument(TypeToken.of(secondType).isAssignableFrom(firstType), + Preconditions.checkArgument(TypeToken.of(secondType).isSupertypeOf(TypeToken.of(firstType)), "Types between stages didn't match. Mismatch between %s -> %s", firstType, secondType); } diff --git a/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/plugin/MetricsOperationTimer.java b/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/plugin/MetricsOperationTimer.java index 54be3ab4c73d..018761b563f4 100644 --- a/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/plugin/MetricsOperationTimer.java +++ b/cdap-app-templates/cdap-etl/cdap-etl-core/src/main/java/io/cdap/cdap/etl/common/plugin/MetricsOperationTimer.java @@ -36,7 +36,7 @@ public class MetricsOperationTimer implements OperationTimer { public MetricsOperationTimer(StageMetrics stageMetrics) { this.stageMetrics = stageMetrics; - this.stopwatch = new Stopwatch(); + this.stopwatch = Stopwatch.createUnstarted(); } /** @@ -65,7 +65,7 @@ public void stop() { */ @Override public void reset() { - emitTimeMetrics(stopwatch.elapsedTime(TimeUnit.MICROSECONDS)); + emitTimeMetrics(stopwatch.elapsed(TimeUnit.MICROSECONDS)); stopwatch.reset(); } diff --git a/cdap-app-templates/cdap-etl/cdap-etl-proto/pom.xml b/cdap-app-templates/cdap-etl/cdap-etl-proto/pom.xml index 1b85b5075989..1156791038ef 100644 --- a/cdap-app-templates/cdap-etl/cdap-etl-proto/pom.xml +++ b/cdap-app-templates/cdap-etl/cdap-etl-proto/pom.xml @@ -58,4 +58,18 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 17 + 17 + + + + + diff --git a/cdap-app-templates/cdap-etl/cdap-etl-tools/pom.xml b/cdap-app-templates/cdap-etl/cdap-etl-tools/pom.xml index ff695b69d2ae..6b47b33fee03 100644 --- a/cdap-app-templates/cdap-etl/cdap-etl-tools/pom.xml +++ b/cdap-app-templates/cdap-etl/cdap-etl-tools/pom.xml @@ -70,9 +70,18 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + maven-shade-plugin - 3.2.1 + 3.5.2 etl-tools-fat-jar diff --git a/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/BatchSQLEngineAdapter.java b/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/BatchSQLEngineAdapter.java index 6349970d4f79..8182e78dcf11 100644 --- a/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/BatchSQLEngineAdapter.java +++ b/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/BatchSQLEngineAdapter.java @@ -16,7 +16,7 @@ package io.cdap.cdap.etl.spark.batch; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import io.cdap.cdap.api.SQLEngineContext; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; @@ -344,7 +344,7 @@ private BatchCollectionFactory pullInternal(SQLDataset dataset) throws SQ SQLPullDataset sqlPullDataset = sqlEngine.getPullProvider(pullRequest); // Run operation to read from the InputFormatProvider supplied by this operation. - ClassLoader classLoader = Objects.firstNonNull(Thread.currentThread().getContextClassLoader(), + ClassLoader classLoader = MoreObjects.firstNonNull(Thread.currentThread().getContextClassLoader(), getClass().getClassLoader()); JavaPairRDD pairRDD = RDDUtils.readUsingInputFormat(jsc, sqlPullDataset, classLoader, Object.class, Object.class); diff --git a/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/DatasetInfoTypeAdapter.java b/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/DatasetInfoTypeAdapter.java index 9b70578c3bec..3499786de6ef 100644 --- a/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/DatasetInfoTypeAdapter.java +++ b/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/DatasetInfoTypeAdapter.java @@ -16,7 +16,7 @@ package io.cdap.cdap.etl.spark.batch; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.reflect.TypeParameter; import com.google.common.reflect.TypeToken; import com.google.gson.JsonDeserializationContext; @@ -48,7 +48,7 @@ public DatasetInfo deserialize(JsonElement json, Type typeOfT, JsonDeserializati return new DatasetInfo(datasetName, datasetArgs, null); } String datasetSplitClass = obj.get("datasetSplitClass").getAsString(); - ClassLoader classLoader = Objects.firstNonNull(Thread.currentThread().getContextClassLoader(), + ClassLoader classLoader = MoreObjects.firstNonNull(Thread.currentThread().getContextClassLoader(), SparkBatchSourceFactory.class.getClassLoader()); try { Class splitClass = classLoader.loadClass(datasetSplitClass); diff --git a/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/SparkBatchSourceFactory.java b/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/SparkBatchSourceFactory.java index e3b14bc18d59..b24e468af84f 100644 --- a/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/SparkBatchSourceFactory.java +++ b/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/SparkBatchSourceFactory.java @@ -16,7 +16,7 @@ package io.cdap.cdap.etl.spark.batch; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import io.cdap.cdap.api.data.batch.Input; import io.cdap.cdap.api.data.batch.InputFormatProvider; import io.cdap.cdap.api.data.batch.Split; @@ -157,7 +157,7 @@ private JavaPairRDD createInputRDD(JavaSparkExecutionContext sec, J if (inputFormatProviders.containsKey(inputName)) { InputFormatProvider inputFormatProvider = inputFormatProviders.get(inputName); - ClassLoader classLoader = Objects.firstNonNull(currentThread().getContextClassLoader(), + ClassLoader classLoader = MoreObjects.firstNonNull(currentThread().getContextClassLoader(), getClass().getClassLoader()); return RDDUtils.readUsingInputFormat(jsc, inputFormatProvider, classLoader, keyClass, diff --git a/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/function/AlertPassFilter.java b/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/function/AlertPassFilter.java index 43dc7cba941d..410e275ba9c1 100644 --- a/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/function/AlertPassFilter.java +++ b/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/function/AlertPassFilter.java @@ -33,6 +33,6 @@ public class AlertPassFilter implements FlatMapFunction, Aler public Iterator call(RecordInfo input) throws Exception { //noinspection unchecked return input.getType() == RecordType.ALERT - ? Iterators.singletonIterator(((Alert) input.getValue())) : Iterators.emptyIterator(); + ? Iterators.singletonIterator(((Alert) input.getValue())) : java.util.Collections.emptyIterator(); } } diff --git a/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/function/ErrorPassFilter.java b/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/function/ErrorPassFilter.java index f70ef7ae5d5f..adccf7adbcf7 100644 --- a/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/function/ErrorPassFilter.java +++ b/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/function/ErrorPassFilter.java @@ -35,6 +35,6 @@ public class ErrorPassFilter implements FlatMapFunction, E public Iterator> call(RecordInfo input) throws Exception { //noinspection unchecked return input.getType() == RecordType.ERROR - ? Iterators.singletonIterator((ErrorRecord) input.getValue()) : Iterators.emptyIterator(); + ? Iterators.singletonIterator((ErrorRecord) input.getValue()) : java.util.Collections.emptyIterator(); } } diff --git a/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/function/OutputPassFilter.java b/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/function/OutputPassFilter.java index 7ad3f930fe83..d89be92823e1 100644 --- a/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/function/OutputPassFilter.java +++ b/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/function/OutputPassFilter.java @@ -44,6 +44,6 @@ public OutputPassFilter(String port) { public Iterator call(RecordInfo input) throws Exception { //noinspection unchecked return input.getType() == RecordType.OUTPUT && Objects.equals(port, input.getFromPort()) - ? Iterators.singletonIterator((T) input.getValue()) : Iterators.emptyIterator(); + ? Iterators.singletonIterator((T) input.getValue()) : java.util.Collections.emptyIterator(); } } diff --git a/cdap-app-templates/cdap-etl/hydrator-test/pom.xml b/cdap-app-templates/cdap-etl/hydrator-test/pom.xml index e91afff1108b..8dc7d77a292d 100644 --- a/cdap-app-templates/cdap-etl/hydrator-test/pom.xml +++ b/cdap-app-templates/cdap-etl/hydrator-test/pom.xml @@ -58,4 +58,18 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + diff --git a/cdap-app-templates/cdap-program-report/pom.xml b/cdap-app-templates/cdap-program-report/pom.xml index 0208c5eb42dd..4a08550a656c 100644 --- a/cdap-app-templates/cdap-program-report/pom.xml +++ b/cdap-app-templates/cdap-program-report/pom.xml @@ -277,6 +277,7 @@ + @@ -308,6 +309,15 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + net.alchim31.maven scala-maven-plugin diff --git a/cdap-app-templates/cdap-program-report/src/main/java/io/cdap/cdap/report/main/RunMetaFileManager.java b/cdap-app-templates/cdap-program-report/src/main/java/io/cdap/cdap/report/main/RunMetaFileManager.java index 48da7d7c8b7b..da3d35f8c752 100644 --- a/cdap-app-templates/cdap-program-report/src/main/java/io/cdap/cdap/report/main/RunMetaFileManager.java +++ b/cdap-app-templates/cdap-program-report/src/main/java/io/cdap/cdap/report/main/RunMetaFileManager.java @@ -108,7 +108,11 @@ public void syncOutputStreamsIfRequired() throws InterruptedException { public void cleanup() { Collection outputStreams = namespaceToLogFileStreamMap.values(); for (RunMetaFileOutputStream outputStream : outputStreams) { - Closeables.closeQuietly(outputStream); + try { + outputStream.close(); + } catch (IOException e) { + // ignore + } } } @@ -149,7 +153,11 @@ private void rotateOutputStreamIfNeeded(RunMetaFileOutputStream runMetaFileOutpu (System.currentTimeMillis() - runMetaFileOutputStream.getCreateTime()) > maxFileOpenDurationMillis; if (runMetaFileOutputStream.getSize() > maxFileSizeBytes || isExpired) { - Closeables.closeQuietly(runMetaFileOutputStream); + try { + runMetaFileOutputStream.close(); + } catch (IOException e) { + // ignore + } createLogFileOutputStreamWithRetry(namespace, timestamp); } } diff --git a/cdap-app-templates/cdap-program-report/src/main/java/io/cdap/cdap/report/main/RunMetaFileOutputStream.java b/cdap-app-templates/cdap-program-report/src/main/java/io/cdap/cdap/report/main/RunMetaFileOutputStream.java index 8f01288af5a6..4e146f20ac58 100644 --- a/cdap-app-templates/cdap-program-report/src/main/java/io/cdap/cdap/report/main/RunMetaFileOutputStream.java +++ b/cdap-app-templates/cdap-program-report/src/main/java/io/cdap/cdap/report/main/RunMetaFileOutputStream.java @@ -62,8 +62,20 @@ class RunMetaFileOutputStream implements Closeable, Flushable { this.createTime = createTime; this.fileSize = 0; } catch (IOException e) { - Closeables.closeQuietly(outputStream); - Closeables.closeQuietly(dataFileWriter); + if (outputStream != null) { + try { + outputStream.close(); + } catch (IOException ex) { + // ignore + } + } + if (dataFileWriter != null) { + try { + dataFileWriter.close(); + } catch (IOException ex) { + // ignore + } + } throw e; } } diff --git a/cdap-authenticator-ext-gcp/pom.xml b/cdap-authenticator-ext-gcp/pom.xml index 23775ae22cba..51bca4b26526 100644 --- a/cdap-authenticator-ext-gcp/pom.xml +++ b/cdap-authenticator-ext-gcp/pom.xml @@ -73,6 +73,20 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + dist @@ -104,7 +118,6 @@ org.apache.maven.plugins maven-jar-plugin - 2.4 jar @@ -112,6 +125,7 @@ ${project.build.directory}/libexec ${project.groupId}.${project.build.finalName} + false jar diff --git a/cdap-cli-tests/pom.xml b/cdap-cli-tests/pom.xml index d6bc0fe640fc..69cc351f1ee9 100644 --- a/cdap-cli-tests/pom.xml +++ b/cdap-cli-tests/pom.xml @@ -138,6 +138,15 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + org.sonatype.central diff --git a/cdap-cli/pom.xml b/cdap-cli/pom.xml index 5a96df00d81d..e55c542babc6 100644 --- a/cdap-cli/pom.xml +++ b/cdap-cli/pom.xml @@ -82,10 +82,19 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 17 + 17 + + org.apache.maven.plugins maven-shade-plugin - 3.2.1 + 3.5.2 package false @@ -95,6 +104,12 @@ ${main.class} + + + com.google.common + io.cdap.cdap.cli.shaded.com.google.common + + @@ -132,7 +147,6 @@ org.apache.maven.plugins maven-jar-plugin - 2.4 default-jar @@ -143,7 +157,7 @@ org.apache.maven.plugins maven-shade-plugin - 3.2.1 + 3.5.2 shade-jar @@ -207,26 +221,7 @@ - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - copy-jar-to-stage-packaging - package - - - - - - - run - - - - + diff --git a/cdap-cli/src/main/java/io/cdap/cdap/cli/CLIMainArgs.java b/cdap-cli/src/main/java/io/cdap/cdap/cli/CLIMainArgs.java index 94ce374a66eb..0c62d3a1ee31 100644 --- a/cdap-cli/src/main/java/io/cdap/cdap/cli/CLIMainArgs.java +++ b/cdap-cli/src/main/java/io/cdap/cdap/cli/CLIMainArgs.java @@ -16,7 +16,7 @@ package io.cdap.cdap.cli; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.collect.Lists; import java.util.Arrays; import java.util.List; @@ -45,7 +45,7 @@ public String[] getCommandTokens() { @Override public int hashCode() { - return Objects.hashCode(Arrays.hashCode(optionTokens), Arrays.hashCode(commandTokens)); + return java.util.Objects.hash(Arrays.hashCode(optionTokens), Arrays.hashCode(commandTokens)); } @Override @@ -63,7 +63,7 @@ public boolean equals(Object obj) { @Override public String toString() { - return Objects.toStringHelper(this).add("optionTokens", Arrays.toString(optionTokens)) + return MoreObjects.toStringHelper(this).add("optionTokens", Arrays.toString(optionTokens)) .add("commandTokens", Arrays.toString(commandTokens)).toString(); } diff --git a/cdap-cli/src/main/java/io/cdap/cdap/cli/ProgramIdArgument.java b/cdap-cli/src/main/java/io/cdap/cdap/cli/ProgramIdArgument.java index dc76a22e3791..535c4a348260 100644 --- a/cdap-cli/src/main/java/io/cdap/cdap/cli/ProgramIdArgument.java +++ b/cdap-cli/src/main/java/io/cdap/cdap/cli/ProgramIdArgument.java @@ -16,7 +16,7 @@ package io.cdap.cdap.cli; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; /** * Represents a program information, such as the app id and program id. @@ -49,18 +49,18 @@ public boolean equals(Object o) { } ProgramIdArgument other = (ProgramIdArgument) o; - return Objects.equal(appId, other.getAppId()) - && Objects.equal(programId, other.getProgramId()); + return java.util.Objects.equals(appId, other.getAppId()) + && java.util.Objects.equals(programId, other.getProgramId()); } @Override public int hashCode() { - return Objects.hashCode(appId, programId); + return java.util.Objects.hash(appId, programId); } @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("appId", appId) .add("programId", programId) .toString(); diff --git a/cdap-cli/src/main/java/io/cdap/cdap/cli/command/system/HelpCommand.java b/cdap-cli/src/main/java/io/cdap/cdap/cli/command/system/HelpCommand.java index ab553fc4a8d2..2a1d684b1e7c 100644 --- a/cdap-cli/src/main/java/io/cdap/cdap/cli/command/system/HelpCommand.java +++ b/cdap-cli/src/main/java/io/cdap/cdap/cli/command/system/HelpCommand.java @@ -16,16 +16,8 @@ package io.cdap.cdap.cli.command.system; -import com.google.common.base.Optional; -import com.google.common.base.Predicate; -import com.google.common.base.Predicates; import com.google.common.base.Splitter; import com.google.common.base.Strings; -import com.google.common.base.Supplier; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import com.google.common.collect.Multimap; import io.cdap.cdap.cli.ArgumentName; import io.cdap.cdap.cli.Categorized; import io.cdap.cdap.cli.CommandCategory; @@ -35,9 +27,16 @@ import io.cdap.common.cli.Command; import io.cdap.common.cli.CommandSet; import java.io.PrintStream; +import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.function.Supplier; + /** * Prints helper text for all commands. @@ -61,13 +60,14 @@ public void execute(Arguments arguments, PrintStream output) throws Exception { if (arguments.hasArgument(ArgumentName.COMMAND_CATEGORY.getName())) { // help with one category - Multimap categorizedCommands = categorizeCommands(commands.get(), + Map> categorizedCommands = categorizeCommands(commands.get(), CommandCategory.GENERAL, - Predicates.alwaysTrue()); + c -> true); String commandCategoryInput = arguments.get(ArgumentName.COMMAND_CATEGORY.getName()); CommandCategory category = CommandCategory.valueOfNameIgnoreCase(commandCategoryInput); - List commandList = Lists.newArrayList(categorizedCommands.get(category.getName())); + List commandList = new ArrayList<>( + categorizedCommands.getOrDefault(category.getName(), Collections.emptyList())); if (commandList.isEmpty()) { output.printf("No commands found in the %s category", category.getName()); output.println(); @@ -78,11 +78,12 @@ public void execute(Arguments arguments, PrintStream output) throws Exception { } else { // normal help - Multimap categorizedCommands = categorizeCommands(commands.get(), + Map> categorizedCommands = categorizeCommands(commands.get(), CommandCategory.GENERAL, - Predicates.alwaysTrue()); + c -> true); for (CommandCategory category : CommandCategory.values()) { - List commandList = Lists.newArrayList(categorizedCommands.get(category.getName())); + List commandList = new ArrayList<>( + categorizedCommands.getOrDefault(category.getName(), Collections.emptyList())); if (commandList.isEmpty()) { continue; } @@ -110,9 +111,9 @@ public int compare(Command command, Command command2) { } } - protected Multimap categorizeCommands(Iterable> commandSets, + protected Map> categorizeCommands(Iterable> commandSets, CommandCategory defaultCategory, Predicate filter) { - Multimap result = HashMultimap.create(); + Map> result = new HashMap<>(); for (CommandSet commandSet : commandSets) { populate(result, commandSet, getCategory(commandSet), defaultCategory, filter); } @@ -122,17 +123,26 @@ protected Multimap categorizeCommands(Iterable result, CommandSet commandSet, + private void populate(Map> result, CommandSet commandSet, Optional parentCategory, CommandCategory defaultCategory, Predicate filter) { - for (Command childCommand : Iterables.filter(commandSet.getCommands(), filter)) { - Optional commandCategory = getCategory(childCommand).or(parentCategory); - result.put(commandCategory.or(defaultCategory.getName()), childCommand); + for (Command childCommand : commandSet.getCommands()) { + if (filter.test(childCommand)) { + Optional commandCategory = getCategory(childCommand); + if (!commandCategory.isPresent()) { + commandCategory = parentCategory; + } + String categoryName = commandCategory.orElse(defaultCategory.getName()); + result.computeIfAbsent(categoryName, k -> new ArrayList<>()).add(childCommand); + } } for (CommandSet childCommandSet : commandSet.getCommandSets()) { - Optional commandCategory = getCategory(childCommandSet).or(parentCategory); + Optional commandCategory = getCategory(childCommandSet); + if (!commandCategory.isPresent()) { + commandCategory = parentCategory; + } populate(result, childCommandSet, commandCategory, defaultCategory, filter); } } @@ -141,7 +151,7 @@ private Optional getCategory(Object object) { if (object instanceof Categorized) { return Optional.of(((Categorized) object).getCategory()); } - return Optional.absent(); + return Optional.empty(); } @Override diff --git a/cdap-cli/src/main/java/io/cdap/cdap/cli/command/system/SearchCommandsCommand.java b/cdap-cli/src/main/java/io/cdap/cdap/cli/command/system/SearchCommandsCommand.java index f902ac21e50a..6358f39aff53 100644 --- a/cdap-cli/src/main/java/io/cdap/cdap/cli/command/system/SearchCommandsCommand.java +++ b/cdap-cli/src/main/java/io/cdap/cdap/cli/command/system/SearchCommandsCommand.java @@ -16,10 +16,6 @@ package io.cdap.cdap.cli.command.system; -import com.google.common.base.Predicate; -import com.google.common.base.Supplier; -import com.google.common.collect.Lists; -import com.google.common.collect.Multimap; import io.cdap.cdap.cli.ArgumentName; import io.cdap.cdap.cli.CommandCategory; import io.cdap.cdap.cli.util.table.TableRendererConfig; @@ -27,8 +23,12 @@ import io.cdap.common.cli.Command; import io.cdap.common.cli.CommandSet; import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; -import javax.annotation.Nullable; +import java.util.Map; +import java.util.function.Predicate; +import java.util.function.Supplier; /** * Searches available commands. @@ -57,15 +57,10 @@ public void execute(Arguments arguments, PrintStream output) throws Exception { } final String query = queryString; - Predicate filter = new Predicate() { - @Override - public boolean apply(@Nullable Command input) { - return input != null && input.getPattern().matches(query); - } - }; + Predicate filter = input -> input != null && input.getPattern().matches(query); output.println(); - Multimap categorizedCommands = categorizeCommands(commands.get(), + Map> categorizedCommands = categorizeCommands(commands.get(), CommandCategory.GENERAL, filter); if (categorizedCommands.isEmpty()) { output.printf("No matches found for \"%s\"", originalQuery); @@ -76,7 +71,8 @@ public boolean apply(@Nullable Command input) { output.println(); for (CommandCategory category : CommandCategory.values()) { - List commandList = Lists.newArrayList(categorizedCommands.get(category.getName())); + List commandList = new ArrayList<>( + categorizedCommands.getOrDefault(category.getName(), Collections.emptyList())); if (commandList.isEmpty()) { continue; } diff --git a/cdap-client-tests/pom.xml b/cdap-client-tests/pom.xml index 8e389b526ef2..9c4660b06822 100644 --- a/cdap-client-tests/pom.xml +++ b/cdap-client-tests/pom.xml @@ -147,6 +147,15 @@ the License. + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + org.sonatype.central diff --git a/cdap-client-tests/src/test/java/io/cdap/cdap/client/util/RESTClientTest.java b/cdap-client-tests/src/test/java/io/cdap/cdap/client/util/RESTClientTest.java index bb8c309874c5..442e7ce65347 100644 --- a/cdap-client-tests/src/test/java/io/cdap/cdap/client/util/RESTClientTest.java +++ b/cdap-client-tests/src/test/java/io/cdap/cdap/client/util/RESTClientTest.java @@ -20,7 +20,7 @@ import static com.google.inject.matcher.Matchers.any; import static com.google.inject.matcher.Matchers.only; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.AbstractIdleService; import com.google.gson.Gson; @@ -72,13 +72,13 @@ public class RESTClientTest { @Before public void setUp() throws IOException { httpService = new TestHttpService(); - httpService.startAndWait(); + httpService.startAsync().awaitRunning(); restClient = new RESTClient(ClientConfig.builder().setUnavailableRetryLimit(3).build()); } @After public void tearDown() { - httpService.stopAndWait(); + httpService.stopAsync().awaitTerminated(); } @Test @@ -245,7 +245,7 @@ protected void shutDown() throws Exception { @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("bindAddress", httpService.getBindAddress()) .toString(); } diff --git a/cdap-client/pom.xml b/cdap-client/pom.xml index d3161c4910ec..4f5eabb57c2e 100644 --- a/cdap-client/pom.xml +++ b/cdap-client/pom.xml @@ -105,5 +105,19 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + diff --git a/cdap-client/src/main/java/io/cdap/cdap/client/config/ConnectionConfig.java b/cdap-client/src/main/java/io/cdap/cdap/client/config/ConnectionConfig.java index b50f7b23f3d2..af69555f6d47 100644 --- a/cdap-client/src/main/java/io/cdap/cdap/client/config/ConnectionConfig.java +++ b/cdap-client/src/main/java/io/cdap/cdap/client/config/ConnectionConfig.java @@ -15,7 +15,7 @@ */ package io.cdap.cdap.client.config; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import io.cdap.cdap.common.conf.CConfiguration; import io.cdap.cdap.common.conf.Constants; @@ -117,7 +117,7 @@ public String getApiPath() { @Override public int hashCode() { - return Objects.hashCode(hostname, port, sslEnabled, apiPath); + return java.util.Objects.hash(hostname, port, sslEnabled, apiPath); } @Override @@ -129,15 +129,15 @@ public boolean equals(Object obj) { return false; } final ConnectionConfig other = (ConnectionConfig) obj; - return Objects.equal(this.hostname, other.hostname) - && Objects.equal(this.port, other.port) - && Objects.equal(this.sslEnabled, other.sslEnabled) - && Objects.equal(this.apiPath, other.apiPath); + return java.util.Objects.equals(this.hostname, other.hostname) + && java.util.Objects.equals(this.port, other.port) + && java.util.Objects.equals(this.sslEnabled, other.sslEnabled) + && java.util.Objects.equals(this.apiPath, other.apiPath); } @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("hostname", hostname) .add("port", port) .add("sslEnabled", sslEnabled) diff --git a/cdap-common-unit-test/pom.xml b/cdap-common-unit-test/pom.xml index c69a45ac19f6..f1c38df701bf 100644 --- a/cdap-common-unit-test/pom.xml +++ b/cdap-common-unit-test/pom.xml @@ -39,4 +39,18 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + diff --git a/cdap-common/bin/functions.sh b/cdap-common/bin/functions.sh index 381c8d955f22..f45d39009385 100644 --- a/cdap-common/bin/functions.sh +++ b/cdap-common/bin/functions.sh @@ -352,6 +352,10 @@ cdap_set_java () { fi export JAVA_VERSION=${__java_version} export JAVA=${__java} + if [ "${__java_version}" -gt 8 ]; then + export JAVA_OPTS="${JAVA_OPTS} --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.management/java.lang.management=ALL-UNNAMED" + export OPTS="${OPTS} --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.management/java.lang.management=ALL-UNNAMED" + fi return 0 } diff --git a/cdap-common/pom.xml b/cdap-common/pom.xml index cc81058e854b..c4adb7bbb745 100644 --- a/cdap-common/pom.xml +++ b/cdap-common/pom.xml @@ -34,6 +34,7 @@ ${maven.build.timestamp} yyyy-MM-dd'T'HH:mm:ss.SSSZ + --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.management/java.lang.management=ALL-UNNAMED @@ -258,10 +259,18 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 17 + 17 + + org.apache.maven.plugins maven-jar-plugin - 2.4 test-jar @@ -271,6 +280,14 @@ + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M6 + + ${argLine} + + diff --git a/cdap-common/src/main/java/com/google/common/base/Stopwatch.java b/cdap-common/src/main/java/com/google/common/base/Stopwatch.java new file mode 100644 index 000000000000..5cbf82d5fb20 --- /dev/null +++ b/cdap-common/src/main/java/com/google/common/base/Stopwatch.java @@ -0,0 +1,161 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.common.base; + +import java.time.Duration; +import java.util.concurrent.TimeUnit; + +/** + * Shadow compatibility class for Guava's Stopwatch to preserve compatibility + * with precompiled dependencies (like Apache Tephra) under Guava 32. + */ +public final class Stopwatch { + private final Ticker ticker; + private boolean isRunning; + private long elapsedNanos; + private long startTick; + + public static Stopwatch createUnstarted() { + return new Stopwatch(); + } + + public static Stopwatch createUnstarted(Ticker ticker) { + return new Stopwatch(ticker); + } + + public static Stopwatch createStarted() { + return new Stopwatch().start(); + } + + public static Stopwatch createStarted(Ticker ticker) { + return new Stopwatch(ticker).start(); + } + + /** + * Legacy constructor made public for binary compatibility with precompiled libraries. + */ + public Stopwatch() { + this.ticker = Ticker.systemTicker(); + } + + /** + * Legacy constructor made public for binary compatibility with precompiled libraries. + */ + public Stopwatch(Ticker ticker) { + this.ticker = Preconditions.checkNotNull(ticker, "ticker"); + } + + public boolean isRunning() { + return isRunning; + } + + public Stopwatch start() { + Preconditions.checkState(!isRunning, "This stopwatch is already running."); + isRunning = true; + startTick = ticker.read(); + return this; + } + + public Stopwatch stop() { + long tick = ticker.read(); + Preconditions.checkState(isRunning, "This stopwatch is already stopped."); + isRunning = false; + elapsedNanos += tick - startTick; + return this; + } + + public Stopwatch reset() { + elapsedNanos = 0; + isRunning = false; + return this; + } + + private long elapsedNanos() { + return isRunning ? ticker.read() - startTick + elapsedNanos : elapsedNanos; + } + + public long elapsed(TimeUnit timeUnit) { + return timeUnit.convert(elapsedNanos(), TimeUnit.NANOSECONDS); + } + + public Duration elapsed() { + return Duration.ofNanos(elapsedNanos()); + } + + // --- Legacy methods for backward compatibility --- + + @Deprecated + public long elapsedTime(TimeUnit timeUnit) { + return elapsed(timeUnit); + } + + @Deprecated + public long elapsedMillis() { + return elapsed(TimeUnit.MILLISECONDS); + } + + @Override + public String toString() { + long nanos = elapsedNanos(); + TimeUnit unit = chooseUnit(nanos); + double value = (double) nanos / TimeUnit.NANOSECONDS.convert(1, unit); + return String.format("%.4g %s", value, abbreviate(unit)); + } + + private static TimeUnit chooseUnit(long nanos) { + if (TimeUnit.DAYS.convert(nanos, TimeUnit.NANOSECONDS) > 0) { + return TimeUnit.DAYS; + } + if (TimeUnit.HOURS.convert(nanos, TimeUnit.NANOSECONDS) > 0) { + return TimeUnit.HOURS; + } + if (TimeUnit.MINUTES.convert(nanos, TimeUnit.NANOSECONDS) > 0) { + return TimeUnit.MINUTES; + } + if (TimeUnit.SECONDS.convert(nanos, TimeUnit.NANOSECONDS) > 0) { + return TimeUnit.SECONDS; + } + if (TimeUnit.MILLISECONDS.convert(nanos, TimeUnit.NANOSECONDS) > 0) { + return TimeUnit.MILLISECONDS; + } + if (TimeUnit.MICROSECONDS.convert(nanos, TimeUnit.NANOSECONDS) > 0) { + return TimeUnit.MICROSECONDS; + } + return TimeUnit.NANOSECONDS; + } + + private static String abbreviate(TimeUnit unit) { + switch (unit) { + case NANOSECONDS: + return "ns"; + case MICROSECONDS: + return "μs"; + case MILLISECONDS: + return "ms"; + case SECONDS: + return "s"; + case MINUTES: + return "min"; + case HOURS: + return "h"; + case DAYS: + return "d"; + default: + throw new AssertionError(); + } + } +} diff --git a/cdap-common/src/main/java/com/google/common/io/InputSupplier.java b/cdap-common/src/main/java/com/google/common/io/InputSupplier.java new file mode 100644 index 000000000000..92e48ac7ae23 --- /dev/null +++ b/cdap-common/src/main/java/com/google/common/io/InputSupplier.java @@ -0,0 +1,29 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.common.io; + +import java.io.IOException; + +/** + * Dummy interface to satisfy compile-time reference resolution of legacy Guava APIs in third-party dependencies. + * + * @param type of input resource + */ +@Deprecated +public interface InputSupplier { + T getInput() throws IOException; +} diff --git a/cdap-common/src/main/java/com/google/common/io/OutputSupplier.java b/cdap-common/src/main/java/com/google/common/io/OutputSupplier.java new file mode 100644 index 000000000000..0e82d6220353 --- /dev/null +++ b/cdap-common/src/main/java/com/google/common/io/OutputSupplier.java @@ -0,0 +1,29 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.common.io; + +import java.io.IOException; + +/** + * Dummy interface to satisfy compile-time reference resolution of legacy Guava APIs in third-party dependencies. + * + * @param type of output resource + */ +@Deprecated +public interface OutputSupplier { + T getOutput() throws IOException; +} diff --git a/cdap-common/src/main/java/com/google/common/util/concurrent/Service.java b/cdap-common/src/main/java/com/google/common/util/concurrent/Service.java new file mode 100644 index 000000000000..6b8ba7f0b9cd --- /dev/null +++ b/cdap-common/src/main/java/com/google/common/util/concurrent/Service.java @@ -0,0 +1,364 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.common.util.concurrent; + +import java.time.Duration; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * Shadow compatibility interface for Guava Service to preserve compatibility with precompiled + * dependencies (like Apache Tephra) under Guava 32. + */ +public interface Service { + default Service startAsync() { + try { + boolean implementsStart = false; + Class clazz = this.getClass(); + while (clazz != null && clazz != Object.class) { + try { + clazz.getDeclaredMethod("start"); + implementsStart = true; + break; + } catch (NoSuchMethodException e) { + clazz = clazz.getSuperclass(); + } + } + if (implementsStart) { + try { + addListener(new Listener() { + @Override + public void failed(State from, Throwable failure) { + FailureTracker.setFailure(Service.this, failure); + } + }, MoreExecutors.directExecutor()); + } catch (Exception e) { + // ignore + } + start(); + return this; + } + } catch (Exception e) { + // fallback + } + throw new UnsupportedOperationException("startAsync() is not implemented by " + this.getClass().getName()); + } + + boolean isRunning(); + State state(); + + default Service stopAsync() { + try { + boolean implementsStop = false; + Class clazz = this.getClass(); + while (clazz != null && clazz != Object.class) { + try { + clazz.getDeclaredMethod("stop"); + implementsStop = true; + break; + } catch (NoSuchMethodException e) { + clazz = clazz.getSuperclass(); + } + } + if (implementsStop) { + stop(); + return this; + } + } catch (Exception e) { + // fallback + } + throw new UnsupportedOperationException("stopAsync() is not implemented by " + this.getClass().getName()); + } + + default void awaitRunning() { + State currentState = state(); + if (currentState == State.RUNNING) { + return; + } + if (currentState == State.FAILED) { + throw new IllegalStateException("Service failed", failureCause()); + } + if (currentState == State.TERMINATED || currentState == State.STOPPING) { + throw new IllegalStateException("Service already stopped or stopping"); + } + + final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); + final Throwable[] failedThrowable = new Throwable[1]; + addListener(new Listener() { + @Override + public void running() { + latch.countDown(); + } + @Override + public void failed(State from, Throwable failure) { + failedThrowable[0] = failure; + latch.countDown(); + } + }, MoreExecutors.directExecutor()); + + State recheckState = state(); + if (recheckState == State.RUNNING) { + return; + } + if (recheckState == State.FAILED) { + throw new IllegalStateException("Service failed", failureCause()); + } + + try { + latch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + + if (failedThrowable[0] != null) { + throw new IllegalStateException("Service failed", failedThrowable[0]); + } + } + + default void awaitRunning(Duration timeout) throws TimeoutException { + awaitRunning(timeout.toNanos(), TimeUnit.NANOSECONDS); + } + + default void awaitRunning(long timeout, TimeUnit unit) throws TimeoutException { + State currentState = state(); + if (currentState == State.RUNNING) { + return; + } + if (currentState == State.FAILED) { + throw new IllegalStateException("Service failed", failureCause()); + } + if (currentState == State.TERMINATED || currentState == State.STOPPING) { + throw new IllegalStateException("Service already stopped or stopping"); + } + + final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); + final Throwable[] failedThrowable = new Throwable[1]; + addListener(new Listener() { + @Override + public void running() { + latch.countDown(); + } + @Override + public void failed(State from, Throwable failure) { + failedThrowable[0] = failure; + latch.countDown(); + } + }, MoreExecutors.directExecutor()); + + State recheckState = state(); + if (recheckState == State.RUNNING) { + return; + } + if (recheckState == State.FAILED) { + throw new IllegalStateException("Service failed", failureCause()); + } + + try { + if (!latch.await(timeout, unit)) { + throw new TimeoutException("Timed out waiting for service to run"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + + if (failedThrowable[0] != null) { + throw new IllegalStateException("Service failed", failedThrowable[0]); + } + } + + default void awaitTerminated() { + State currentState = state(); + if (currentState == State.TERMINATED) { + return; + } + if (currentState == State.FAILED) { + throw new IllegalStateException("Service failed", failureCause()); + } + + final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); + final Throwable[] failedThrowable = new Throwable[1]; + addListener(new Listener() { + @Override + public void terminated(State from) { + latch.countDown(); + } + @Override + public void failed(State from, Throwable failure) { + failedThrowable[0] = failure; + latch.countDown(); + } + }, MoreExecutors.directExecutor()); + + State recheckState = state(); + if (recheckState == State.TERMINATED) { + return; + } + if (recheckState == State.FAILED) { + throw new IllegalStateException("Service failed", failureCause()); + } + + try { + latch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + + if (failedThrowable[0] != null) { + throw new IllegalStateException("Service failed", failedThrowable[0]); + } + } + + default void awaitTerminated(Duration timeout) throws TimeoutException { + awaitTerminated(timeout.toNanos(), TimeUnit.NANOSECONDS); + } + + default void awaitTerminated(long timeout, TimeUnit unit) throws TimeoutException { + State currentState = state(); + if (currentState == State.TERMINATED) { + return; + } + if (currentState == State.FAILED) { + throw new IllegalStateException("Service failed", failureCause()); + } + + final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); + final Throwable[] failedThrowable = new Throwable[1]; + addListener(new Listener() { + @Override + public void terminated(State from) { + latch.countDown(); + } + @Override + public void failed(State from, Throwable failure) { + failedThrowable[0] = failure; + latch.countDown(); + } + }, MoreExecutors.directExecutor()); + + State recheckState = state(); + if (recheckState == State.TERMINATED) { + return; + } + if (recheckState == State.FAILED) { + throw new IllegalStateException("Service failed", failureCause()); + } + + try { + if (!latch.await(timeout, unit)) { + throw new TimeoutException("Timed out waiting for service to terminate"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + + if (failedThrowable[0] != null) { + throw new IllegalStateException("Service failed", failedThrowable[0]); + } + } + + default Throwable failureCause() { + Throwable t = FailureTracker.getFailure(this); + if (t != null) { + return t; + } + return new IllegalStateException("Service failed with unknown cause"); + } + + void addListener(Listener listener, Executor executor); + + enum State { + NEW, + STARTING, + RUNNING, + STOPPING, + TERMINATED, + FAILED + } + + abstract class Listener { + public Listener() {} + public void starting() {} + public void running() {} + public void stopping(State from) {} + public void terminated(State from) {} + public void failed(State from, Throwable failure) {} + } + + // --- Legacy backward compatibility default methods --- + + default ListenableFuture start() { + final SettableFuture future = SettableFuture.create(); + addListener(new Listener() { + @Override + public void running() { + future.set(State.RUNNING); + } + + @Override + public void failed(State from, Throwable failure) { + future.setException(failure); + } + }, MoreExecutors.directExecutor()); + startAsync(); + return future; + } + + default State startAndWait() { + startAsync().awaitRunning(); + return state(); + } + + default ListenableFuture stop() { + final SettableFuture future = SettableFuture.create(); + addListener(new Listener() { + @Override + public void terminated(State from) { + future.set(State.TERMINATED); + } + + @Override + public void failed(State from, Throwable failure) { + future.setException(failure); + } + }, MoreExecutors.directExecutor()); + stopAsync(); + return future; + } + + default State stopAndWait() { + stopAsync().awaitTerminated(); + return state(); + } + + class FailureTracker { + private static final java.util.Map failures = + java.util.Collections.synchronizedMap(new java.util.WeakHashMap<>()); + + public static void setFailure(Service service, Throwable failure) { + failures.put(service, failure); + } + + public static Throwable getFailure(Service service) { + return failures.get(service); + } + } +} diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/HttpExceptionHandler.java b/cdap-common/src/main/java/io/cdap/cdap/common/HttpExceptionHandler.java index 5e345694fd8b..a688b88e8f9c 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/HttpExceptionHandler.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/HttpExceptionHandler.java @@ -16,7 +16,7 @@ package io.cdap.cdap.common; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.base.Throwables; import io.cdap.cdap.api.common.HttpErrorStatusProvider; import io.cdap.cdap.api.service.ServiceUnavailableException; @@ -76,7 +76,7 @@ public void handle(Throwable t, HttpRequest request, HttpResponder responder) { // If it is not some known exception type, response with 500. LOG.error("Unexpected error: request={} {} user={}:", request.method().name(), request.getUri(), - Objects.firstNonNull(SecurityRequestContext.getUserId(), ""), t); + MoreObjects.firstNonNull(SecurityRequestContext.getUserId(), ""), t); responder.sendString(HttpResponseStatus.INTERNAL_SERVER_ERROR, Throwables.getRootCause(t).getMessage()); } @@ -84,6 +84,6 @@ public void handle(Throwable t, HttpRequest request, HttpResponder responder) { private void logWithTrace(HttpRequest request, Throwable t) { LOG.trace("Error in handling request={} {} for user={}:", request.method().name(), request.getUri(), - Objects.firstNonNull(SecurityRequestContext.getUserId(), ""), t); + MoreObjects.firstNonNull(SecurityRequestContext.getUserId(), ""), t); } } diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/dataset/DatasetClassRewriter.java b/cdap-common/src/main/java/io/cdap/cdap/common/dataset/DatasetClassRewriter.java index a760a008cfd8..cb4d1e84d3e5 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/dataset/DatasetClassRewriter.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/dataset/DatasetClassRewriter.java @@ -100,7 +100,7 @@ private final class DatasetClassVisitor extends ClassVisitor { private boolean interfaceClass; private DatasetClassVisitor(String className, ClassWriter cw) { - super(Opcodes.ASM5, cw); + super(Opcodes.ASM9, cw); this.datasetType = Type.getObjectType(className.replace('.', '/')); this.fields = new HashSet<>(); } @@ -137,7 +137,7 @@ public MethodVisitor visitMethod(int access, final String name, DATASET_RUNTIME_CONTEXT_TYPE.getDescriptor(), null, null); } - return new FinallyAdapter(Opcodes.ASM5, mv, access, name, desc) { + return new FinallyAdapter(Opcodes.ASM9, mv, access, name, desc) { private boolean hasRead; private boolean hasWrite; diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/guice/KafkaClientModule.java b/cdap-common/src/main/java/io/cdap/cdap/common/guice/KafkaClientModule.java index a90f9d361a51..6fc55279528c 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/guice/KafkaClientModule.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/guice/KafkaClientModule.java @@ -129,7 +129,8 @@ public ZKClientService get() { // The logic doesn't need to be sophisticated since it is a private binding and only used by the // wrapping KafkaClientService and BrokerService, which they will make sure no duplicate calls will be // made to the start/stop methods. - return new ForwardingZKClientService(zkClientService) { + final ZKClientService finalZkClientService = zkClientService; + return new ForwardingZKClientService(finalZkClientService) { @Override public ListenableFuture start() { if (startedCount.getAndIncrement() == 0) { @@ -145,6 +146,51 @@ public ListenableFuture stop() { } return Futures.immediateFuture(State.TERMINATED); } + + @Override + public Throwable failureCause() { + return finalZkClientService.failureCause(); + } + + @Override + public void awaitTerminated() { + if (startedCount.get() == 0) { + finalZkClientService.awaitTerminated(); + } + } + + @Override + public void awaitTerminated(long timeout, TimeUnit unit) throws java.util.concurrent.TimeoutException { + if (startedCount.get() == 0) { + finalZkClientService.awaitTerminated(timeout, unit); + } + } + + @Override + public void awaitRunning() { + finalZkClientService.awaitRunning(); + } + + @Override + public void awaitRunning(long timeout, TimeUnit unit) throws java.util.concurrent.TimeoutException { + finalZkClientService.awaitRunning(timeout, unit); + } + + @Override + public com.google.common.util.concurrent.Service startAsync() { + if (startedCount.getAndIncrement() == 0) { + finalZkClientService.startAsync(); + } + return this; + } + + @Override + public com.google.common.util.concurrent.Service stopAsync() { + if (startedCount.decrementAndGet() == 0) { + finalZkClientService.stopAsync(); + } + return this; + } }; } } @@ -166,12 +212,12 @@ private abstract static class AbstractServiceWithZkClient ext @Override protected final void startUp() throws Exception { - zkClientService.startAndWait(); + zkClientService.startAsync().awaitRunning(); try { - delegate.startAndWait(); + delegate.startAsync().awaitRunning(); } catch (Exception e) { try { - zkClientService.stopAndWait(); + zkClientService.stopAsync().awaitTerminated(); } catch (Exception se) { e.addSuppressed(se); } @@ -182,16 +228,16 @@ protected final void startUp() throws Exception { @Override protected final void shutDown() throws Exception { try { - delegate.stopAndWait(); + delegate.stopAsync().awaitTerminated(); } catch (Exception e) { try { - zkClientService.stopAndWait(); + zkClientService.stopAsync().awaitTerminated(); } catch (Exception se) { e.addSuppressed(se); } throw e; } - zkClientService.stopAndWait(); + zkClientService.stopAsync().awaitTerminated(); } protected T getDelegate() { diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/http/AbstractBodyConsumer.java b/cdap-common/src/main/java/io/cdap/cdap/common/http/AbstractBodyConsumer.java index 9543e33fa80c..7b52543d7fff 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/http/AbstractBodyConsumer.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/http/AbstractBodyConsumer.java @@ -73,8 +73,12 @@ public final void finished(HttpResponder responder) { public final void handleError(Throwable cause) { try { LOG.error("Failed to handle upload", cause); - if (output != null) { - Closeables.closeQuietly(output); + try { + if (output != null) { + output.close(); + } + } catch (IOException e) { + // Ignore } onError(cause); // The netty-http framework will response with 500, no need to response in here. diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/http/SpillableBodyConsumer.java b/cdap-common/src/main/java/io/cdap/cdap/common/http/SpillableBodyConsumer.java index 10ee554be82f..7d5baf7ce764 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/http/SpillableBodyConsumer.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/http/SpillableBodyConsumer.java @@ -88,7 +88,13 @@ public void chunk(ByteBuf request, HttpResponder responder) { @Override public void finished(HttpResponder responder) { - Closeables.closeQuietly(outputStream); + try { + if (outputStream != null) { + outputStream.close(); + } + } catch (IOException e) { + // Ignore + } try (InputStream is = new CombineInputStream(buffer, outputStream == null ? null : spillPath)) { processInput(is, responder); @@ -103,7 +109,13 @@ public void finished(HttpResponder responder) { @Override public void handleError(Throwable cause) { - Closeables.closeQuietly(outputStream); + try { + if (outputStream != null) { + outputStream.close(); + } + } catch (IOException e) { + // Ignore + } cleanup(); } diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/internal/guava/ClassPath.java b/cdap-common/src/main/java/io/cdap/cdap/common/internal/guava/ClassPath.java index 2f7d0e9c1fd9..1947a1539259 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/internal/guava/ClassPath.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/internal/guava/ClassPath.java @@ -328,7 +328,7 @@ public String getSimpleName() { String innerClassName = className.substring(lastDollarSign + 1); // local and anonymous classes are prefixed with number (1,2,3...), anonymous classes are // entirely numeric whereas local classes have the user supplied name as a suffix - return CharMatcher.DIGIT.trimLeadingFrom(innerClassName); + return CharMatcher.digit().trimLeadingFrom(innerClassName); } String packageName = getPackageName(); if (packageName.isEmpty()) { @@ -395,6 +395,26 @@ static ImmutableMap getClassPathEntries(ClassLoader classloade entries.put(uri, classloader); } } + } else if (classloader == ClassLoader.getSystemClassLoader()) { + // Fallback for Java 9+ where System ClassLoader is not a URLClassLoader + String classpath = System.getProperty("java.class.path"); + if (classpath != null) { + for (String path : Splitter.on(File.pathSeparator).split(classpath)) { + try { + File file = new File(path); + URI uri = file.toURI(); + if (!predicate.apply(uri)) { + continue; + } + if (!entries.containsKey(uri)) { + entries.put(uri, classloader); + } + } catch (Exception e) { + // Ignore bad entry + LOG.warn("Failed to parse classpath entry: " + path, e); + } + } + } } return ImmutableMap.copyOf(entries); } diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/io/DFSSeekableInputStream.java b/cdap-common/src/main/java/io/cdap/cdap/common/io/DFSSeekableInputStream.java index 16ca5b46db1c..f2cff23c2302 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/io/DFSSeekableInputStream.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/io/DFSSeekableInputStream.java @@ -69,7 +69,11 @@ public void close() throws IOException { super.close(); } finally { if (sizeProvider instanceof Closeable) { - Closeables.closeQuietly((Closeable) sizeProvider); + try { + ((Closeable) sizeProvider).close(); + } catch (IOException e) { + // Ignore + } } } } diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/io/DefaultCachingPathProvider.java b/cdap-common/src/main/java/io/cdap/cdap/common/io/DefaultCachingPathProvider.java index 15e0c8988c43..18d0002c39a4 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/io/DefaultCachingPathProvider.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/io/DefaultCachingPathProvider.java @@ -123,7 +123,8 @@ void clearCache(String fileName, long lastModified) { } String getCacheName(Location location) { - return Hashing.md5().hashString(location.toURI().getPath()).toString() + "-" + return Hashing.md5().hashString(location.toURI().getPath(), + java.nio.charset.StandardCharsets.UTF_8).toString() + "-" + location.getName(); } diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/io/Locations.java b/cdap-common/src/main/java/io/cdap/cdap/common/io/Locations.java index 4eb04253b402..63c6ee0860e6 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/io/Locations.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/io/Locations.java @@ -21,9 +21,8 @@ import com.google.common.base.Throwables; import com.google.common.io.ByteStreams; import com.google.common.io.Closeables; -import com.google.common.io.InputSupplier; -import com.google.common.io.OutputSupplier; import io.cdap.cdap.common.lang.FunctionWithException; +import io.cdap.cdap.common.lang.ThrowingSupplier; import io.cdap.cdap.common.lang.jar.BundleJarUtil; import io.cdap.cdap.common.utils.DirUtils; import io.cdap.cdap.common.utils.FileUtils; @@ -105,11 +104,11 @@ public int compare(Location o1, Location o2) { * requested. * @return A {@link InputSupplier}. */ - public static InputSupplier newInputSupplier(final FileSystem fs, + public static ThrowingSupplier newInputSupplier(final FileSystem fs, final Path path) { - return new InputSupplier() { + return new ThrowingSupplier() { @Override - public SeekableInputStream getInput() throws IOException { + public SeekableInputStream get() throws IOException { FSDataInputStream input = fs.open(path); try { return new DFSSeekableInputStream(input, @@ -130,11 +129,11 @@ public SeekableInputStream getInput() throws IOException { * @param location Location for the input stream. * @return A {@link InputSupplier}. */ - public static InputSupplier newInputSupplier( + public static ThrowingSupplier newInputSupplier( final Location location) { - return new InputSupplier() { + return new ThrowingSupplier() { @Override - public SeekableInputStream getInput() throws IOException { + public SeekableInputStream get() throws IOException { InputStream input = location.getInputStream(); try { if (input instanceof FileInputStream) { @@ -343,7 +342,9 @@ private static void expandTarStream(TarArchiveInputStream tis, File targetDir) DirUtils.mkdirs(output); } else { DirUtils.mkdirs(output.getParentFile()); - ByteStreams.copy(tis, com.google.common.io.Files.newOutputStreamSupplier(output)); + try (OutputStream os = Files.newOutputStream(output.toPath())) { + ByteStreams.copy(tis, os); + } } entry = tis.getNextTarEntry(); } @@ -418,7 +419,7 @@ public T next() throws IOException { * @param location Location for the output. * @return A {@link OutputSupplier}. */ - public static OutputSupplier newOutputSupplier(final Location location) { + public static ThrowingSupplier newOutputSupplier(final Location location) { return location::getOutputStream; } diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/lang/ClassLoaders.java b/cdap-common/src/main/java/io/cdap/cdap/common/lang/ClassLoaders.java index dfc46ccabd46..e7604cde8a09 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/lang/ClassLoaders.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/lang/ClassLoaders.java @@ -16,7 +16,7 @@ package io.cdap.cdap.common.lang; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.base.Splitter; import com.google.common.base.Throwables; import java.io.File; @@ -27,6 +27,7 @@ import java.net.URL; import java.net.URLClassLoader; import java.net.URLConnection; +import java.util.ArrayList; import java.util.Collection; import java.util.Deque; import java.util.Iterator; @@ -67,7 +68,7 @@ private ClassLoaders() { */ public static Class loadClass(String className, @Nullable ClassLoader classLoader, Object caller) throws ClassNotFoundException { - ClassLoader cl = Objects.firstNonNull(classLoader, caller.getClass().getClassLoader()); + ClassLoader cl = MoreObjects.firstNonNull(classLoader, caller.getClass().getClassLoader()); return cl.loadClass(className); } @@ -173,15 +174,25 @@ public static > T getClassLoaderURLs(ClassLoad */ public static > T getClassLoaderURLs(ClassLoader classLoader, boolean childFirst, T urls) { - Deque classLoaders = collectURLClassLoaders(classLoader, new LinkedList<>()); + Deque classLoaders = collectClassLoaders(classLoader, new LinkedList<>()); - Iterator iterator = + Iterator iterator = childFirst ? classLoaders.iterator() : classLoaders.descendingIterator(); while (iterator.hasNext()) { - URLClassLoader cl = iterator.next(); - for (URL url : cl.getURLs()) { - if (urls.add(url) && (url.getProtocol().equals("file"))) { - addClassPathFromJar(url, urls); + ClassLoader cl = iterator.next(); + if (cl instanceof URLClassLoader) { + for (URL url : ((URLClassLoader) cl).getURLs()) { + if (urls.add(url) && (url.getProtocol().equals("file"))) { + addClassPathFromJar(url, urls); + } + } + } else if (cl == ClassLoader.getSystemClassLoader()) { + // For Java 9+, the system classloader is not a URLClassLoader. + // We retrieve the URLs from "java.class.path" system property. + for (URL url : getSystemClassLoaderURLs()) { + if (urls.add(url) && (url.getProtocol().equals("file"))) { + addClassPathFromJar(url, urls); + } } } } @@ -190,16 +201,16 @@ public static > T getClassLoaderURLs(ClassLoad } /** - * Collects {@link URLClassLoader} along the {@link ClassLoader} chain. This method recognizes + * Collects {@link ClassLoader} along the {@link ClassLoader} chain. This method recognizes * both {@link Delegator} and {@link CombineClassLoader} and will unfold the delegating - * classloaders inside it. The order of insertion + * classloaders inside it. * * @param classLoader the classloader to start the search from * @param result a collection for storing the result * @param type of the collection * @return the result collection */ - private static > T collectURLClassLoaders( + private static > T collectClassLoaders( ClassLoader classLoader, T result) { // Do BFS from the bottom of the ClassLoader chain @@ -226,8 +237,8 @@ private static > T collectURLClassL // Use add first for delegate, which effectively is replacing the current classloader queue.addFirst((ClassLoader) delegate); } - } else if (cl instanceof URLClassLoader) { - result.add((URLClassLoader) cl); + } else { + result.add(cl); } if (cl.getParent() != null) { @@ -238,6 +249,21 @@ private static > T collectURLClassL return result; } + private static List getSystemClassLoaderURLs() { + List urls = new ArrayList<>(); + String classpath = System.getProperty("java.class.path"); + if (classpath != null) { + for (String path : Splitter.on(File.pathSeparatorChar).omitEmptyStrings().split(classpath)) { + try { + urls.add(new File(path).toURI().toURL()); + } catch (MalformedURLException e) { + // ignore + } + } + } + return urls; + } + /** * Creates a {@link Function} that perform class name to class resource lookup. * diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/lang/ClassPathResources.java b/cdap-common/src/main/java/io/cdap/cdap/common/lang/ClassPathResources.java index 501e746223da..d332d94db42e 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/lang/ClassPathResources.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/lang/ClassPathResources.java @@ -160,12 +160,21 @@ static ClassAcceptor createClassAcceptor(ClassLoader classLoader, Collection>> ClassPathResources ACCEPT: " + className); // Ignore platform classes if (platformClassloader.getResource(className.replace('.', '/') + ".class") != null) { return false; } + // Ignore classes that are known to trigger ASM7 failures under Java 17 + // due to PermittedSubclasses attributes in enums or relocated libraries. + if (className.startsWith("io.cdap.cdap.internal.guava.") + || className.startsWith("io.cdap.cdap.api.dataset.lib.Partitioning") + || className.startsWith("io.cdap.cdap.proto.security.StandardPermission")) { + return false; + } + // Should ignore classes from SLF4J implementation, otherwise it will include logback lib, // which shouldn't be visible through the program classloader. if (className.startsWith("org.slf4j.impl.")) { @@ -236,7 +245,16 @@ private static > T findClassDependencies( final ClassLoader classLoader, Iterable classes, final T result) throws IOException { - Dependencies.findClassDependencies(classLoader, createClassAcceptor(classLoader, result), + ClassLoader wrappingClassLoader = new ClassLoader(classLoader) { + @Override + public java.io.InputStream getResourceAsStream(String name) { + if (name.endsWith(".class")) { + System.out.println(">>> ClassLoader getResourceAsStream: " + name); + } + return super.getResourceAsStream(name); + } + }; + Dependencies.findClassDependencies(wrappingClassLoader, createClassAcceptor(classLoader, result), classes); return result; } diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/lang/GuavaClassRewriter.java b/cdap-common/src/main/java/io/cdap/cdap/common/lang/GuavaClassRewriter.java index 87a848708337..48096377e5e7 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/lang/GuavaClassRewriter.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/lang/GuavaClassRewriter.java @@ -133,7 +133,7 @@ private byte[] rewritePreconditions(InputStream input) throws IOException { ClassReader cr = new ClassReader(input); ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_FRAMES); - cr.accept(new PreconditionsRewriter(Opcodes.ASM7, cw, methods), ClassReader.EXPAND_FRAMES); + cr.accept(new PreconditionsRewriter(Opcodes.ASM9, cw, methods), ClassReader.EXPAND_FRAMES); return cw.toByteArray(); } @@ -149,7 +149,7 @@ private byte[] rewriteMoreExecutors(InputStream input) throws IOException { ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_FRAMES); Method method = new Method("directExecutor", Type.getType(Executor.class), new Type[0]); - cr.accept(new ClassVisitor(Opcodes.ASM7, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM9, cw) { private boolean hasMethod; diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/leveldb/LevelDBClassRewriter.java b/cdap-common/src/main/java/io/cdap/cdap/common/leveldb/LevelDBClassRewriter.java index e68dfc46dd0e..24850bd6ac59 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/leveldb/LevelDBClassRewriter.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/leveldb/LevelDBClassRewriter.java @@ -96,7 +96,7 @@ public byte[] rewriteClass(String className, InputStream input) throws IOExcepti ClassReader cr = new ClassReader(input); ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_FRAMES); - cr.accept(new ClassVisitor(Opcodes.ASM7, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM9, cw) { @Override public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { @@ -105,7 +105,7 @@ public MethodVisitor visitMethod(int access, String name, String descriptor, if (!FILE_META_DATA_CONSTRUCTOR.equals(new Method(name, descriptor))) { return mv; } - return new AdviceAdapter(Opcodes.ASM7, mv, access, name, descriptor) { + return new AdviceAdapter(Opcodes.ASM9, mv, access, name, descriptor) { @Override protected void onMethodEnter() { loadArg(SMALLEST_KEY_PARAM); diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/logging/AbstractLoggingContext.java b/cdap-common/src/main/java/io/cdap/cdap/common/logging/AbstractLoggingContext.java index 11f95f46aaa0..fe88d0b2d816 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/logging/AbstractLoggingContext.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/logging/AbstractLoggingContext.java @@ -16,7 +16,7 @@ package io.cdap.cdap.common.logging; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.collect.Maps; import java.lang.reflect.Method; import java.util.Collection; @@ -113,7 +113,7 @@ public Map getSystemTagsAsString() { @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("systemTags", systemTags) .toString(); } @@ -140,7 +140,7 @@ public String getValue() { @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("name", name) .add("value", value) .toString(); diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/resource/ResourceBalancerService.java b/cdap-common/src/main/java/io/cdap/cdap/common/resource/ResourceBalancerService.java index 42283b6e1fcf..52bc30d7b8e2 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/resource/ResourceBalancerService.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/resource/ResourceBalancerService.java @@ -94,13 +94,13 @@ public void leader() { coordinator = new ResourceCoordinator(zk, discoveryServiceClient, new BalancedAssignmentStrategy()); - coordinator.startAndWait(); + coordinator.startAsync().awaitRunning(); } @Override public void follower() { if (coordinator != null) { - coordinator.stopAndWait(); + coordinator.stopAsync().awaitTerminated(); coordinator = null; } } @@ -130,8 +130,8 @@ protected void startUp() throws Exception { Discoverable discoverable = createDiscoverable(serviceName); cancelDiscoverable = discoveryService.register(ResolvingDiscoverable.of(discoverable)); - election.start(); - resourceClient.startAndWait(); + election.startAsync().awaitRunning(); + resourceClient.startAsync().awaitRunning(); cancelResourceHandler = resourceClient.subscribe(serviceName, createResourceHandler(discoverable)); @@ -181,18 +181,18 @@ public void onChange(Collection partitionReplicas) { LOG.info("Partitions changed {}, service: {}", partitions, serviceName); try { if (service != null) { - service.stopAndWait(); + service.stopAsync().awaitTerminated(); } if (partitions.isEmpty() || !election.isRunning()) { service = null; } else { service = createService(partitions); - service.startAndWait(); + service.startAsync().awaitRunning(); } } catch (Throwable t) { LOG.error("Failed to change partitions, service: {}.", serviceName, t); completion.setException(t); - stop(); + stopAsync(); } } @@ -200,7 +200,7 @@ public void onChange(Collection partitionReplicas) { public void finished(Throwable failureCause) { try { if (service != null) { - service.stopAndWait(); + service.stopAsync().awaitTerminated(); service = null; } completion.set(null); diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/security/AuthEnforceRewriter.java b/cdap-common/src/main/java/io/cdap/cdap/common/security/AuthEnforceRewriter.java index bf29bada05ed..b51a4c8abd3e 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/security/AuthEnforceRewriter.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/security/AuthEnforceRewriter.java @@ -182,7 +182,7 @@ private static final class AuthEnforceAnnotationVisitor extends ClassVisitor { private boolean interfaceClass; AuthEnforceAnnotationVisitor(String className) { - super(Opcodes.ASM5); + super(Opcodes.ASM9); this.className = className; this.methodAnnotations = new HashMap<>(); this.fieldDetails = new HashMap<>(); @@ -221,7 +221,7 @@ public MethodVisitor visitMethod(final int access, final String methodName, // AuthEnforce annotation details and also sets a boolean flag hasEnforce which is used later in // visitParameterAnnotation to visit the annotations on parameters of this method only if it had AuthEnforce // annotation. - return new MethodVisitor(Opcodes.ASM5) { + return new MethodVisitor(Opcodes.ASM9) { final Map parameterDetails = new HashMap<>(); AnnotationNode authEnforceAnnotationNode; @@ -388,7 +388,7 @@ private final class AuthEnforceAnnotationRewriter extends ClassVisitor { AuthEnforceAnnotationRewriter(String className, ClassWriter cw, Map fieldDetails, Map methodAnnotations) { - super(Opcodes.ASM5, cw); + super(Opcodes.ASM9, cw); this.className = className; this.classType = Type.getObjectType(className.replace(".", "/")); this.methodAnnotations = methodAnnotations; @@ -409,7 +409,7 @@ public MethodVisitor visitMethod(int access, final String methodName, final Stri if (annotationDetail == null) { return mv; } - return new AdviceAdapter(Opcodes.ASM5, mv, access, methodName, methodDesc) { + return new AdviceAdapter(Opcodes.ASM9, mv, access, methodName, methodDesc) { @Override protected void onMethodEnter() { LOG.trace( diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/service/AbstractRetryableScheduledService.java b/cdap-common/src/main/java/io/cdap/cdap/common/service/AbstractRetryableScheduledService.java index 47a52840859f..49b6675ee621 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/service/AbstractRetryableScheduledService.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/service/AbstractRetryableScheduledService.java @@ -17,6 +17,7 @@ package io.cdap.cdap.common.service; import com.google.common.util.concurrent.AbstractScheduledService; +import com.google.common.util.concurrent.Service; import io.cdap.cdap.api.retry.RetriesExhaustedException; import io.cdap.cdap.common.logging.LogSamplers; import io.cdap.cdap.common.logging.Loggers; @@ -56,7 +57,7 @@ public abstract class AbstractRetryableScheduledService extends AbstractSchedule */ protected AbstractRetryableScheduledService(RetryStrategy retryStrategy) { this.retryStrategy = retryStrategy; - addListener(new ServiceListenerAdapter() { + addListener(new Service.Listener() { @Override public void failed(State from, Throwable failure) { LOG.error("Scheduled service {} terminated due to failure", getServiceName(), failure); diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/service/RetryOnStartFailureService.java b/cdap-common/src/main/java/io/cdap/cdap/common/service/RetryOnStartFailureService.java index 55f616d5020e..a719bd498534 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/service/RetryOnStartFailureService.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/service/RetryOnStartFailureService.java @@ -20,6 +20,7 @@ import com.google.common.util.concurrent.AbstractService; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.Service; import com.google.common.util.concurrent.Uninterruptibles; import java.util.concurrent.TimeUnit; @@ -68,7 +69,29 @@ public void run() { while (!stopped) { try { - currentDelegate.start().get(); + java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); + java.util.concurrent.atomic.AtomicReference failure = + new java.util.concurrent.atomic.AtomicReference<>(); + currentDelegate.addListener(new Service.Listener() { + @Override + public void running() { + latch.countDown(); + } + + @Override + public void failed(Service.State from, Throwable failureCause) { + failure.set(failureCause); + latch.countDown(); + } + }, MoreExecutors.directExecutor()); + + currentDelegate.startAsync(); + latch.await(); + + if (failure.get() != null) { + throw failure.get(); + } + // Only assigned the delegate if and only if the delegate service started successfully startedService = currentDelegate; break; @@ -112,25 +135,42 @@ protected void doStop() { // the setting of the startedService field. When that happens, the stop failure state is not propagated. // Nevertheless, there won't be any service left behind without stopping. if (startedService != null) { - Futures.addCallback(startedService.stop(), new FutureCallback() { + startedService.addListener(new Service.Listener() { @Override - public void onSuccess(State result) { + public void terminated(Service.State from) { notifyStopped(); } @Override - public void onFailure(Throwable t) { - notifyFailed(t); + public void failed(Service.State from, Throwable failure) { + notifyFailed(failure); } - }, Threads.SAME_THREAD_EXECUTOR); + }, MoreExecutors.directExecutor()); + startedService.stopAsync(); return; } - // If there is no started service, stop the current delete, but no need to propagate the stop state + // If there is no started service, stop the current delegate, but no need to propagate the stop state // because if the underlying service is not yet started due to failure, it shouldn't affect the stop state // of this retrying service. if (currentDelegate != null) { - currentDelegate.stop().addListener(this::notifyStopped, Threads.SAME_THREAD_EXECUTOR); + Service.State state = currentDelegate.state(); + if (state == Service.State.FAILED || state == Service.State.TERMINATED) { + notifyStopped(); + return; + } + currentDelegate.addListener(new Service.Listener() { + @Override + public void terminated(Service.State from) { + notifyStopped(); + } + + @Override + public void failed(Service.State from, Throwable failure) { + notifyStopped(); + } + }, MoreExecutors.directExecutor()); + currentDelegate.stopAsync(); return; } diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/service/Services.java b/cdap-common/src/main/java/io/cdap/cdap/common/service/Services.java index 74d619454692..c8cd448e82fe 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/service/Services.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/service/Services.java @@ -51,9 +51,8 @@ private Services() { public static void startAndWait(Service service, long timeout, TimeUnit timeoutUnit, @Nullable String timeoutErrorMessage) throws TimeoutException, InterruptedException, ExecutionException { - ListenableFuture startFuture = service.start(); try { - startFuture.get(timeout, timeoutUnit); + service.startAsync().awaitRunning(timeout, timeoutUnit); } catch (TimeoutException e) { LOG.error(timeoutErrorMessage != null ? timeoutErrorMessage : "Timeout while waiting to start service.", e); @@ -62,19 +61,19 @@ public static void startAndWait(Service service, long timeout, TimeUnit timeoutU timeoutException.setStackTrace(e.getStackTrace()); } try { - service.stop(); + service.stopAsync(); } catch (Exception stopException) { LOG.error("Error while trying to stop service: ", stopException); } throw timeoutException; - } catch (InterruptedException e) { - LOG.error("Interrupted while waiting to start service.", e); + } catch (IllegalStateException e) { + LOG.error("Service failed to start.", e); try { - service.stop(); + service.stopAsync(); } catch (Exception stopException) { LOG.error("Error while trying to stop service:", stopException); } - throw e; + throw new ExecutionException(e); } } diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/twill/AbstractMasterTwillRunnable.java b/cdap-common/src/main/java/io/cdap/cdap/common/twill/AbstractMasterTwillRunnable.java index d9fd6c75729c..d345eb921439 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/twill/AbstractMasterTwillRunnable.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/twill/AbstractMasterTwillRunnable.java @@ -153,7 +153,7 @@ public void destroy() { private Service.Listener createServiceListener(final String name, final SettableFuture future) { - return new ServiceListenerAdapter() { + return new Service.Listener() { @Override public void terminated(Service.State from) { LOG.info("Service " + name + " terminated"); diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/twill/NoopTwillController.java b/cdap-common/src/main/java/io/cdap/cdap/common/twill/NoopTwillController.java index 3074529dcf19..101312c5ba26 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/twill/NoopTwillController.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/twill/NoopTwillController.java @@ -23,26 +23,36 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.ExecutionException; import javax.annotation.Nullable; import org.apache.twill.api.Command; import org.apache.twill.api.ResourceReport; +import org.apache.twill.api.RunId; import org.apache.twill.api.TwillController; +import org.apache.twill.api.ServiceController; import org.apache.twill.api.logging.LogEntry; import org.apache.twill.api.logging.LogHandler; import org.apache.twill.common.Cancellable; import org.apache.twill.discovery.Discoverable; import org.apache.twill.discovery.ServiceDiscovered; -import org.apache.twill.internal.AbstractExecutionServiceController; import org.apache.twill.internal.RunIds; /** * A no-op {@link TwillController}. */ -final class NoopTwillController extends AbstractExecutionServiceController implements - TwillController { +final class NoopTwillController implements TwillController { + + private final RunId runId; NoopTwillController() { - super(RunIds.generate()); + this.runId = RunIds.generate(); + } + + @Override + public RunId getRunId() { + return runId; } @Override @@ -158,27 +168,53 @@ public Future resetRunnableLogLevels(String runnableName, String... lo } @Override - protected void startUp() { - // no-op + public Future sendCommand(Command command) { + return CompletableFuture.completedFuture(command); } @Override - protected void shutDown() { - // no-op + public Future sendCommand(String runnableName, Command command) { + return CompletableFuture.completedFuture(command); } @Override - public Future sendCommand(Command command) { - return CompletableFuture.completedFuture(command); + public Future terminate() { + return CompletableFuture.completedFuture(this); } @Override - public Future sendCommand(String runnableName, Command command) { - return CompletableFuture.completedFuture(command); + public Future terminate(long gracefulTimeout, TimeUnit gracefulTimeoutUnit) { + return CompletableFuture.completedFuture(this); } @Override public void kill() { - terminate(); + // no-op + } + + @Override + public void onRunning(Runnable runnable, Executor executor) { + executor.execute(runnable); + } + + @Override + public void onTerminated(Runnable runnable, Executor executor) { + executor.execute(runnable); + } + + @Override + public void awaitTerminated() throws ExecutionException { + // no-op + } + + @Override + public void awaitTerminated(long timeout, TimeUnit timeoutUnit) throws TimeoutException, ExecutionException { + // no-op + } + + @Nullable + @Override + public TerminationStatus getTerminationStatus() { + return TerminationStatus.SUCCEEDED; } } diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/utils/ImmutablePair.java b/cdap-common/src/main/java/io/cdap/cdap/common/utils/ImmutablePair.java index 6e3e9d69f9a8..81357066c2a3 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/utils/ImmutablePair.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/utils/ImmutablePair.java @@ -16,7 +16,8 @@ package io.cdap.cdap.common.utils; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; +import java.util.Objects; /** * An {@link ImmutablePair} consists of two elements within. The elements once set in the @@ -83,7 +84,7 @@ public B getSecond() { */ @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("first", first) .add("second", second) .toString(); @@ -96,7 +97,7 @@ public String toString() { */ @Override public int hashCode() { - return Objects.hashCode(first, second); + return Objects.hash(first, second); } /** @@ -114,6 +115,6 @@ public boolean equals(Object o) { return false; } ImmutablePair other = (ImmutablePair) o; - return Objects.equal(first, other.first) && Objects.equal(second, other.second); + return Objects.equals(first, other.first) && Objects.equals(second, other.second); } } diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/utils/TimeBoundIterator.java b/cdap-common/src/main/java/io/cdap/cdap/common/utils/TimeBoundIterator.java index 4598513fe11b..bbbd2cc78850 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/utils/TimeBoundIterator.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/utils/TimeBoundIterator.java @@ -21,6 +21,7 @@ import com.google.common.base.Stopwatch; import com.google.common.collect.AbstractIterator; import java.util.Iterator; +import java.util.concurrent.TimeUnit; /** * An iterator that will act as if there are no more elements if a certain amount of time has @@ -35,7 +36,7 @@ public class TimeBoundIterator extends AbstractIterator { private final Stopwatch stopwatch; public TimeBoundIterator(Iterator delegate, long timeBoundMillis) { - this(delegate, timeBoundMillis, new Stopwatch()); + this(delegate, timeBoundMillis, Stopwatch.createUnstarted()); } public TimeBoundIterator(Iterator delegate, long timeBoundMillis, Stopwatch stopwatch) { @@ -49,7 +50,7 @@ public TimeBoundIterator(Iterator delegate, long timeBoundMillis, Stopwatch s @Override protected T computeNext() { - if (stopwatch.elapsedMillis() < timeBoundMillis && delegate.hasNext()) { + if (stopwatch.elapsed(TimeUnit.MILLISECONDS) < timeBoundMillis && delegate.hasNext()) { return delegate.next(); } return endOfData(); diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/zookeeper/coordination/PartitionReplica.java b/cdap-common/src/main/java/io/cdap/cdap/common/zookeeper/coordination/PartitionReplica.java index f9aae9efccdb..18a34e1904fb 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/zookeeper/coordination/PartitionReplica.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/zookeeper/coordination/PartitionReplica.java @@ -15,8 +15,9 @@ */ package io.cdap.cdap.common.zookeeper.coordination; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.primitives.Ints; +import java.util.Objects; import java.util.Comparator; /** @@ -77,12 +78,12 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hashCode(name, replicaId); + return Objects.hash(name, replicaId); } @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("partition", name) .add("replica", replicaId) .toString(); diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/zookeeper/coordination/ResourceCoordinatorClient.java b/cdap-common/src/main/java/io/cdap/cdap/common/zookeeper/coordination/ResourceCoordinatorClient.java index 01aeb7996ad7..c58ddd7ccc68 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/zookeeper/coordination/ResourceCoordinatorClient.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/zookeeper/coordination/ResourceCoordinatorClient.java @@ -28,6 +28,7 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import io.cdap.cdap.api.common.Bytes; import io.cdap.cdap.common.zookeeper.ZKExtOperations; import java.util.EnumSet; @@ -132,7 +133,8 @@ public ListenableFuture fetchRequirement(String resourceNam return Futures.transform( ZKOperations.ignoreError(zkClient.getData(zkPath), KeeperException.NoNodeException.class, null), - NODE_DATA_TO_REQUIREMENT + NODE_DATA_TO_REQUIREMENT, + MoreExecutors.directExecutor() ); } @@ -150,7 +152,8 @@ public ListenableFuture deleteRequirement(String resourceName) { return Futures.transform( ZKOperations.ignoreError(zkClient.delete(zkPath), KeeperException.NoNodeException.class, resourceName), - Functions.constant(resourceName) + Functions.constant(resourceName), + MoreExecutors.directExecutor() ); } diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/zookeeper/coordination/ResourceRequirement.java b/cdap-common/src/main/java/io/cdap/cdap/common/zookeeper/coordination/ResourceRequirement.java index 15ad26371137..42ecb69df3b8 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/zookeeper/coordination/ResourceRequirement.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/zookeeper/coordination/ResourceRequirement.java @@ -15,9 +15,10 @@ */ package io.cdap.cdap.common.zookeeper.coordination; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSortedSet; +import java.util.Objects; import com.google.common.collect.Maps; import com.google.common.primitives.Ints; import java.util.Map; @@ -60,7 +61,7 @@ public Set getPartitions() { @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("name", name) .add("partitions", partitions) .toString(); @@ -81,7 +82,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hashCode(name, partitions); + return Objects.hash(name, partitions); } /** @@ -134,12 +135,12 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hashCode(name, replicas); + return Objects.hash(name, replicas); } @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("name", name) .add("replicas", replicas) .toString(); diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/zookeeper/election/LeaderElectionInfoService.java b/cdap-common/src/main/java/io/cdap/cdap/common/zookeeper/election/LeaderElectionInfoService.java index 15fa2d73582f..c04094cb9be7 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/zookeeper/election/LeaderElectionInfoService.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/zookeeper/election/LeaderElectionInfoService.java @@ -22,6 +22,7 @@ import com.google.common.util.concurrent.AbstractIdleService; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; import java.nio.charset.StandardCharsets; import java.util.Collections; @@ -81,9 +82,9 @@ public LeaderElectionInfoService(ZKClient zkClient, String leaderElectionPath) { public SortedMap getParticipants(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException { try { - Stopwatch stopwatch = new Stopwatch().start(); + Stopwatch stopwatch = Stopwatch.createStarted(); CountDownLatch readyLatch = readyFuture.get(timeout, unit); - long latchTimeout = Math.max(0, stopwatch.elapsedTime(unit) - timeout); + long latchTimeout = Math.max(0, stopwatch.elapsed(unit) - timeout); readyLatch.await(latchTimeout, unit); } catch (ExecutionException e) { // The ready future never throw on get. If this happen, just return an empty map @@ -237,7 +238,7 @@ public void onFailure(Throwable t) { readyLatch.countDown(); } } - }); + }, MoreExecutors.directExecutor()); } /** diff --git a/cdap-common/src/main/java/io/cdap/cdap/data2/util/TableId.java b/cdap-common/src/main/java/io/cdap/cdap/data2/util/TableId.java index 35c5e51fc9ee..5c4664c3ee47 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/data2/util/TableId.java +++ b/cdap-common/src/main/java/io/cdap/cdap/data2/util/TableId.java @@ -16,9 +16,10 @@ package io.cdap.cdap.data2.util; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import java.util.Objects; /** * Identifier for an HBase and LevelDB tables that contains a namespace and a table name @@ -59,18 +60,18 @@ public boolean equals(Object o) { } TableId that = (TableId) o; - return Objects.equal(namespace, that.getNamespace()) && Objects.equal(tableName, + return Objects.equals(namespace, that.getNamespace()) && Objects.equals(tableName, that.getTableName()); } @Override public int hashCode() { - return Objects.hashCode(namespace, tableName); + return Objects.hash(namespace, tableName); } @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("namespace", namespace) .add("tableName", tableName) .toString(); diff --git a/cdap-common/src/main/java/io/cdap/cdap/internal/app/store/RunRecordDetail.java b/cdap-common/src/main/java/io/cdap/cdap/internal/app/store/RunRecordDetail.java index a2b5d7e945a9..481c6fe62e6c 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/internal/app/store/RunRecordDetail.java +++ b/cdap-common/src/main/java/io/cdap/cdap/internal/app/store/RunRecordDetail.java @@ -15,8 +15,9 @@ */ package io.cdap.cdap.internal.app.store; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.gson.Gson; +import java.util.Objects; import com.google.gson.annotations.SerializedName; import com.google.gson.reflect.TypeToken; import io.cdap.cdap.api.artifact.ArtifactId; @@ -158,27 +159,27 @@ public boolean equals(Object o) { } RunRecordDetail that = (RunRecordDetail) o; - return Objects.equal(this.getProgramRunId(), that.getProgramRunId()) - && Objects.equal(this.getStartTs(), that.getStartTs()) - && Objects.equal(this.getRunTs(), that.getRunTs()) - && Objects.equal(this.getStopTs(), that.getStopTs()) - && Objects.equal(this.getSuspendTs(), that.getSuspendTs()) - && Objects.equal(this.getResumeTs(), that.getResumeTs()) - && Objects.equal(this.getStoppingTs(), that.getStoppingTs()) - && Objects.equal(this.getTerminateTs(), that.getTerminateTs()) - && Objects.equal(this.getStatus(), that.getStatus()) - && Objects.equal(this.getProperties(), that.getProperties()) - && Objects.equal(this.getPeerName(), that.getPeerName()) - && Objects.equal(this.getTwillRunId(), that.getTwillRunId()) + return Objects.equals(this.getProgramRunId(), that.getProgramRunId()) + && Objects.equals(this.getStartTs(), that.getStartTs()) + && Objects.equals(this.getRunTs(), that.getRunTs()) + && Objects.equals(this.getStopTs(), that.getStopTs()) + && Objects.equals(this.getSuspendTs(), that.getSuspendTs()) + && Objects.equals(this.getResumeTs(), that.getResumeTs()) + && Objects.equals(this.getStoppingTs(), that.getStoppingTs()) + && Objects.equals(this.getTerminateTs(), that.getTerminateTs()) + && Objects.equals(this.getStatus(), that.getStatus()) + && Objects.equals(this.getProperties(), that.getProperties()) + && Objects.equals(this.getPeerName(), that.getPeerName()) + && Objects.equals(this.getTwillRunId(), that.getTwillRunId()) && Arrays.equals(this.getSourceId(), that.getSourceId()) - && Objects.equal(this.getArtifactId(), that.getArtifactId()) - && Objects.equal(this.getPrincipal(), that.getPrincipal()) - && Objects.equal(this.getFlowControlStatus(), that.getFlowControlStatus()); + && Objects.equals(this.getArtifactId(), that.getArtifactId()) + && Objects.equals(this.getPrincipal(), that.getPrincipal()) + && Objects.equals(this.getFlowControlStatus(), that.getFlowControlStatus()); } @Override public int hashCode() { - return Objects.hashCode(getProgramRunId(), getStartTs(), getRunTs(), getStopTs(), + return Objects.hash(getProgramRunId(), getStartTs(), getRunTs(), getStopTs(), getSuspendTs(), getResumeTs(), getStoppingTs(), getTerminateTs(), getStatus(), getProperties(), getPeerName(), getTwillRunId(), Arrays.hashCode(getSourceId()), getArtifactId(), getPrincipal(), getFlowControlStatus()); @@ -186,7 +187,7 @@ public int hashCode() { @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("programRunId", getProgramRunId()) .add("startTs", getStartTs()) .add("runTs", getRunTs()) diff --git a/cdap-common/src/main/java/io/cdap/cdap/internal/asm/Classes.java b/cdap-common/src/main/java/io/cdap/cdap/internal/asm/Classes.java index fefdc89e35bc..4892a08c763a 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/internal/asm/Classes.java +++ b/cdap-common/src/main/java/io/cdap/cdap/internal/asm/Classes.java @@ -118,7 +118,7 @@ public static byte[] rewriteMethodToNoop(final String className, InputStream byteCodeStream, final Set methods) throws IOException { ClassReader cr = new ClassReader(byteCodeStream); ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_MAXS); - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM9, cw) { @Override public MethodVisitor visitMethod(final int access, final String name, final String desc, String signature, String[] exceptions) { @@ -141,7 +141,7 @@ public MethodVisitor visitMethod(final int access, final String name, final Stri // VisitMaxs with 0 so that COMPUTE_MAXS from ClassWriter will compute the right values. adapter.visitMaxs(0, 0); - return new MethodVisitor(Opcodes.ASM5) { + return new MethodVisitor(Opcodes.ASM9) { }; } }, 0); diff --git a/cdap-common/src/main/java/org/apache/twill/internal/ServiceListenerAdapter.java b/cdap-common/src/main/java/org/apache/twill/internal/ServiceListenerAdapter.java new file mode 100644 index 000000000000..ce7bb0812c98 --- /dev/null +++ b/cdap-common/src/main/java/org/apache/twill/internal/ServiceListenerAdapter.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.twill.internal; + +import com.google.common.util.concurrent.Service; + +/** + * A shim for {@code org.apache.twill.internal.ServiceListenerAdapter} that extends + * {@code com.google.common.util.concurrent.Service.Listener} (which is an abstract class + * in Guava 32+) instead of implementing it (which was an interface in legacy Guava). + * This resolves {@link IncompatibleClassChangeError} at runtime with Twill. + */ +public abstract class ServiceListenerAdapter extends Service.Listener { + + public ServiceListenerAdapter() { + super(); + } + + @Override + public void starting() { + // No-op + } + + @Override + public void running() { + // No-op + } + + @Override + public void stopping(Service.State from) { + // No-op + } + + @Override + public void terminated(Service.State from) { + // No-op + } + + @Override + public void failed(Service.State from, Throwable failure) { + // No-op + } +} diff --git a/cdap-common/src/main/java/org/apache/twill/internal/zookeeper/DefaultZKClientService.java b/cdap-common/src/main/java/org/apache/twill/internal/zookeeper/DefaultZKClientService.java new file mode 100644 index 000000000000..2886d3d5e369 --- /dev/null +++ b/cdap-common/src/main/java/org/apache/twill/internal/zookeeper/DefaultZKClientService.java @@ -0,0 +1,705 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.twill.internal.zookeeper; + +import com.google.common.base.Preconditions; +import com.google.common.base.Supplier; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.Multimap; +import com.google.common.util.concurrent.AbstractService; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.Service; +import org.apache.twill.common.Cancellable; +import org.apache.twill.common.Threads; +import org.apache.twill.zookeeper.ACLData; +import org.apache.twill.zookeeper.AbstractZKClient; +import org.apache.twill.zookeeper.NodeChildren; +import org.apache.twill.zookeeper.NodeData; +import org.apache.twill.zookeeper.OperationFuture; +import org.apache.twill.zookeeper.ZKClientService; +import org.apache.zookeeper.AsyncCallback; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Stat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; +import javax.annotation.Nullable; + +/** + * The base implementation of {@link ZKClientService}. + */ +public final class DefaultZKClientService extends AbstractZKClient implements ZKClientService { + + private static final Logger LOG = LoggerFactory.getLogger(DefaultZKClientService.class); + + private final String zkStr; + private final int sessionTimeout; + private final List connectionWatchers; + private final Multimap authInfos; + private final AtomicReference zooKeeper; + private final Service serviceDelegate; + private ExecutorService eventExecutor; + + /** + * Creates a new instance. + * @deprecated Use {@link ZKClientService.Builder} instead. + */ + @Deprecated + @SuppressWarnings("unused") + public DefaultZKClientService(String zkStr, int sessionTimeout, Watcher connectionWatcher) { + this(zkStr, sessionTimeout, connectionWatcher, ImmutableMultimap.of()); + } + + public DefaultZKClientService(String zkStr, int sessionTimeout, + Watcher connectionWatcher, Multimap authInfos) { + this.zkStr = zkStr; + this.sessionTimeout = sessionTimeout; + this.connectionWatchers = new CopyOnWriteArrayList<>(); + this.authInfos = copyAuthInfo(authInfos); + addConnectionWatcher(connectionWatcher); + + this.zooKeeper = new AtomicReference<>(); + serviceDelegate = new ServiceDelegate(); + } + + @Override + public Long getSessionId() { + ZooKeeper zk = zooKeeper.get(); + return zk == null ? null : zk.getSessionId(); + } + + @Override + public String getConnectString() { + return zkStr; + } + + @Override + public Cancellable addConnectionWatcher(final Watcher watcher) { + if (watcher == null) { + return new Cancellable() { + @Override + public void cancel() { + // No-op + } + }; + } + + // Invocation of connection watchers are already done inside the event thread, + // hence no need to wrap the watcher again. + connectionWatchers.add(watcher); + return new Cancellable() { + @Override + public void cancel() { + connectionWatchers.remove(watcher); + } + }; + } + + @Override + public OperationFuture create(String path, @Nullable byte[] data, CreateMode createMode, + boolean createParent, Iterable acl) { + return doCreate(path, data, createMode, createParent, ImmutableList.copyOf(acl), false); + } + + private OperationFuture doCreate(final String path, + @Nullable final byte[] data, + final CreateMode createMode, + final boolean createParent, + final List acl, + final boolean ignoreNodeExists) { + final SettableOperationFuture createFuture = SettableOperationFuture.create(path, eventExecutor); + getZooKeeper().create(path, data, acl, createMode, Callbacks.STRING, createFuture); + if (!createParent) { + return createFuture; + } + + // If create parent is request, return a different future + final SettableOperationFuture result = SettableOperationFuture.create(path, eventExecutor); + // Watch for changes in the original future + Futures.addCallback(createFuture, new FutureCallback() { + @Override + public void onSuccess(String path) { + // Propagate if creation was successful + result.set(path); + } + + @Override + public void onFailure(Throwable t) { + // See if the failure can be handled + if (updateFailureResult(t, result, path, ignoreNodeExists)) { + return; + } + // Create the parent node + String parentPath = getParent(path); + if (parentPath.isEmpty()) { + result.setException(t); + return; + } + // Watch for parent creation complete. Parent is created with the unsafe ACL. + Futures.addCallback(doCreate(parentPath, null, CreateMode.PERSISTENT, + true, ZooDefs.Ids.OPEN_ACL_UNSAFE, true), new FutureCallback() { + @Override + public void onSuccess(String parentPath) { + // Create the requested path again + Futures.addCallback( + doCreate(path, data, createMode, false, acl, ignoreNodeExists), new FutureCallback() { + @Override + public void onSuccess(String pathResult) { + result.set(pathResult); + } + + @Override + public void onFailure(Throwable t) { + // handle the failure + updateFailureResult(t, result, path, ignoreNodeExists); + } + }, MoreExecutors.directExecutor()); + } + + @Override + public void onFailure(Throwable t) { + result.setException(t); + } + }, MoreExecutors.directExecutor()); + } + + /** + * Updates the result future based on the given {@link Throwable}. + * @param t Cause of the failure + * @param result Future to be updated + * @param path Request path for the operation + * @return {@code true} if it is a failure, {@code false} otherwise. + */ + private boolean updateFailureResult(Throwable t, SettableOperationFuture result, + String path, boolean ignoreNodeExists) { + // Propagate if there is error + if (!(t instanceof KeeperException)) { + result.setException(t); + return true; + } + KeeperException.Code code = ((KeeperException) t).code(); + // Node already exists, simply return success if it allows for ignoring node exists (for parent node creation). + if (ignoreNodeExists && code == KeeperException.Code.NODEEXISTS) { + // The requested path could be used because it only applies to non-sequential node + result.set(path); + return false; + } + if (code != KeeperException.Code.NONODE) { + result.setException(t); + return true; + } + return false; + } + + /** + * Gets the parent of the given path. + * @param path Path for computing its parent + * @return Parent of the given path, or empty string if the given path is the root path already. + */ + private String getParent(String path) { + String parentPath = path.substring(0, path.lastIndexOf('/')); + return (parentPath.isEmpty() && !"/".equals(path)) ? "/" : parentPath; + } + }, MoreExecutors.directExecutor()); + + return result; + } + + @Override + public OperationFuture exists(String path, Watcher watcher) { + SettableOperationFuture result = SettableOperationFuture.create(path, eventExecutor); + getZooKeeper().exists(path, wrapWatcher(watcher), Callbacks.STAT_NONODE, result); + return result; + } + + @Override + public OperationFuture getChildren(String path, Watcher watcher) { + SettableOperationFuture result = SettableOperationFuture.create(path, eventExecutor); + getZooKeeper().getChildren(path, wrapWatcher(watcher), Callbacks.CHILDREN, result); + return result; + } + + @Override + public OperationFuture getData(String path, Watcher watcher) { + SettableOperationFuture result = SettableOperationFuture.create(path, eventExecutor); + getZooKeeper().getData(path, wrapWatcher(watcher), Callbacks.DATA, result); + + return result; + } + + @Override + public OperationFuture setData(String dataPath, byte[] data, int version) { + SettableOperationFuture result = SettableOperationFuture.create(dataPath, eventExecutor); + getZooKeeper().setData(dataPath, data, version, Callbacks.STAT, result); + return result; + } + + @Override + public OperationFuture delete(String deletePath, int version) { + SettableOperationFuture result = SettableOperationFuture.create(deletePath, eventExecutor); + getZooKeeper().delete(deletePath, version, Callbacks.VOID, result); + return result; + } + + @Override + public OperationFuture getACL(String path) { + SettableOperationFuture result = SettableOperationFuture.create(path, eventExecutor); + getZooKeeper().getACL(path, new Stat(), Callbacks.ACL, result); + return result; + } + + @Override + public OperationFuture setACL(String path, Iterable acl, int version) { + SettableOperationFuture result = SettableOperationFuture.create(path, eventExecutor); + getZooKeeper().setACL(path, ImmutableList.copyOf(acl), version, Callbacks.STAT, result); + return result; + } + + @Override + public Supplier getZooKeeperSupplier() { + return new Supplier() { + @Override + public ZooKeeper get() { + return getZooKeeper(); + } + }; + } + + @Override + public ListenableFuture start() { + return serviceDelegate.start(); + } + + @Override + public State startAndWait() { + return serviceDelegate.startAndWait(); + } + + @Override + public boolean isRunning() { + return serviceDelegate.isRunning(); + } + + @Override + public State state() { + return serviceDelegate.state(); + } + + @Override + public ListenableFuture stop() { + return serviceDelegate.stop(); + } + + @Override + public State stopAndWait() { + return serviceDelegate.stopAndWait(); + } + + @Override + public void addListener(Listener listener, Executor executor) { + serviceDelegate.addListener(listener, executor); + } + + @Override + public Throwable failureCause() { + return serviceDelegate.failureCause(); + } + + @Override + public Service startAsync() { + serviceDelegate.startAsync(); + return this; + } + + @Override + public void awaitRunning() { + serviceDelegate.awaitRunning(); + } + + @Override + public void awaitRunning(long timeout, TimeUnit unit) throws TimeoutException { + serviceDelegate.awaitRunning(timeout, unit); + } + + @Override + public Service stopAsync() { + serviceDelegate.stopAsync(); + return this; + } + + @Override + public void awaitTerminated() { + serviceDelegate.awaitTerminated(); + } + + @Override + public void awaitTerminated(long timeout, TimeUnit unit) throws TimeoutException { + serviceDelegate.awaitTerminated(timeout, unit); + } + + /** + * @return Current {@link ZooKeeper} client. + */ + private ZooKeeper getZooKeeper() { + ZooKeeper zk = zooKeeper.get(); + Preconditions.checkArgument(zk != null, "Not connected to zooKeeper."); + return zk; + } + + /** + * Wraps the given watcher to be called from the event executor. + * @param watcher Watcher to be wrapped + * @return The wrapped Watcher + */ + private Watcher wrapWatcher(final Watcher watcher) { + if (watcher == null) { + return null; + } + return new Watcher() { + @Override + public void process(final WatchedEvent event) { + if (eventExecutor.isShutdown()) { + LOG.debug("Already shutdown. Discarding event: {}", event); + return; + } + eventExecutor.execute(new Runnable() { + @Override + public void run() { + try { + watcher.process(event); + } catch (Throwable t) { + LOG.error("Watcher throws exception.", t); + } + } + }); + } + }; + } + + /** + * Creates a deep copy of the given authInfos multimap. + */ + private Multimap copyAuthInfo(Multimap authInfos) { + Multimap result = ArrayListMultimap.create(); + + for (Map.Entry entry : authInfos.entries()) { + byte[] info = entry.getValue(); + result.put(entry.getKey(), info == null ? null : Arrays.copyOf(info, info.length)); + } + + return result; + } + + private final class ServiceDelegate extends AbstractService implements Watcher { + + private final Runnable stopTask; + + private ServiceDelegate() { + // Creates the stop task runnable in constructor so that if the stop() method is called from shutdown hook, + // it won't fail with class loading error due to failure to load inner class from the shutdown thread. + this.stopTask = createStopTask(); + + // Add a listener for state changes so that we can terminate the service even it is in STARTING state upoon + // stop is requested + addListener(new Listener() { + @Override + public void starting() { + // no-op + } + + @Override + public void running() { + // no-op + } + + @Override + public void stopping(State from) { + if (from == State.STARTING) { + // If it is still starting, just notify that it's started to transit out of the STARTING phase. + notifyStarted(); + } + } + + @Override + public void terminated(State from) { + // no-op + } + + @Override + public void failed(State from, Throwable failure) { + eventExecutor.shutdownNow(); + // Close the ZK client if there is exception. It is needed because the stop task may not get executed + closeZooKeeper(zooKeeper.getAndSet(null)); + } + }, Threads.SAME_THREAD_EXECUTOR); + } + + @Override + protected void doStart() { + // A single thread executor for all events + ThreadPoolExecutor executor = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue(), + Threads.createDaemonThreadFactory("zk-client-EventThread")); + // Just discard the execution if the executor is closed + executor.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy()); + eventExecutor = executor; + + try { + zooKeeper.set(createZooKeeper()); + } catch (IOException e) { + notifyFailed(e); + } + } + + @Override + protected void doStop() { + // Submit a task to the executor to make sure all pending events in the executor are fired before + // transiting this Service into STOPPED state + eventExecutor.submit(stopTask); + eventExecutor.shutdown(); + } + + @Override + public void process(WatchedEvent event) { + State state = state(); + if (state == State.TERMINATED || state == State.FAILED) { + return; + } + + try { + if (event.getState() == Event.KeeperState.SyncConnected && state == State.STARTING) { + LOG.debug("Connected to ZooKeeper: {}", zkStr); + notifyStarted(); + return; + } + if (event.getState() == Event.KeeperState.Expired) { + LOG.info("ZooKeeper session expired: {}", zkStr); + + // When connection expired, simply reconnect again + if (state != State.RUNNING) { + return; + } + eventExecutor.submit(new Runnable() { + @Override + public void run() { + // Only reconnect if the current state is running + if (state() != State.RUNNING) { + return; + } + try { + LOG.info("Reconnect to ZooKeeper due to expiration: {}", zkStr); + closeZooKeeper(zooKeeper.getAndSet(createZooKeeper())); + } catch (IOException e) { + notifyFailed(e); + } + } + }); + } + } finally { + if (event.getType() == Event.EventType.None) { + for (Watcher connectionWatcher : connectionWatchers) { + connectionWatcher.process(event); + } + } + } + } + + + /** + * Creates a {@link Runnable} task that will get executed in the event executor for transiting this + * Service into STOPPED state. + */ + private Runnable createStopTask() { + return new Runnable() { + @Override + public void run() { + try { + // Close the ZK connection in this task will make sure if there is ZK connection created + // after doStop() was called but before this task has been executed is also closed. + // It is possible to happen when the following sequence happens: + // + // 1. session expired, hence the expired event is triggered + // 2. The reconnect task executed. With Service.state() == RUNNING, it creates a new ZK client + // 3. Service.stop() gets called, Service.state() changed to STOPPING + // 4. The new ZK client created from the reconnect thread update the zooKeeper with the new one + closeZooKeeper(zooKeeper.getAndSet(null)); + notifyStopped(); + } catch (Exception e) { + notifyFailed(e); + } + } + }; + } + + /** + * Creates a new ZooKeeper connection. + */ + private ZooKeeper createZooKeeper() throws IOException { + ZooKeeper zk = new ZooKeeper(zkStr, sessionTimeout, wrapWatcher(this)); + for (Map.Entry authInfo : authInfos.entries()) { + zk.addAuthInfo(authInfo.getKey(), authInfo.getValue()); + } + return zk; + } + + /** + * Closes the given {@link ZooKeeper} if it is not null. If there is InterruptedException, + * it will get logged. + */ + private void closeZooKeeper(@Nullable ZooKeeper zk) { + try { + if (zk != null) { + zk.close(); + } + } catch (InterruptedException e) { + LOG.warn("Interrupted when closing ZooKeeper", e); + Thread.currentThread().interrupt(); + } + } + } + + /** + * Collection of generic callbacks that simply reflect results into OperationFuture. + */ + private static final class Callbacks { + static final AsyncCallback.StringCallback STRING = new AsyncCallback.StringCallback() { + @Override + @SuppressWarnings("unchecked") + public void processResult(int rc, String path, Object ctx, String name) { + SettableOperationFuture result = (SettableOperationFuture) ctx; + KeeperException.Code code = KeeperException.Code.get(rc); + if (code == KeeperException.Code.OK) { + result.set((name == null || name.isEmpty()) ? path : name); + return; + } + result.setException(KeeperException.create(code, result.getRequestPath())); + } + }; + + static final AsyncCallback.StatCallback STAT = new AsyncCallback.StatCallback() { + @Override + @SuppressWarnings("unchecked") + public void processResult(int rc, String path, Object ctx, Stat stat) { + SettableOperationFuture result = (SettableOperationFuture) ctx; + KeeperException.Code code = KeeperException.Code.get(rc); + if (code == KeeperException.Code.OK) { + result.set(stat); + return; + } + result.setException(KeeperException.create(code, result.getRequestPath())); + } + }; + + /** + * A stat callback that treats NONODE as success. + */ + static final AsyncCallback.StatCallback STAT_NONODE = new AsyncCallback.StatCallback() { + @Override + @SuppressWarnings("unchecked") + public void processResult(int rc, String path, Object ctx, Stat stat) { + SettableOperationFuture result = (SettableOperationFuture) ctx; + KeeperException.Code code = KeeperException.Code.get(rc); + if (code == KeeperException.Code.OK || code == KeeperException.Code.NONODE) { + result.set(stat); + return; + } + result.setException(KeeperException.create(code, result.getRequestPath())); + } + }; + + static final AsyncCallback.Children2Callback CHILDREN = new AsyncCallback.Children2Callback() { + @Override + @SuppressWarnings("unchecked") + public void processResult(int rc, String path, Object ctx, List children, Stat stat) { + SettableOperationFuture result = (SettableOperationFuture) ctx; + KeeperException.Code code = KeeperException.Code.get(rc); + if (code == KeeperException.Code.OK) { + result.set(new BasicNodeChildren(children, stat)); + return; + } + result.setException(KeeperException.create(code, result.getRequestPath())); + } + }; + + static final AsyncCallback.DataCallback DATA = new AsyncCallback.DataCallback() { + @Override + @SuppressWarnings("unchecked") + public void processResult(int rc, String path, Object ctx, byte[] data, Stat stat) { + SettableOperationFuture result = (SettableOperationFuture) ctx; + KeeperException.Code code = KeeperException.Code.get(rc); + if (code == KeeperException.Code.OK) { + result.set(new BasicNodeData(data, stat)); + return; + } + result.setException(KeeperException.create(code, result.getRequestPath())); + } + }; + + static final AsyncCallback.VoidCallback VOID = new AsyncCallback.VoidCallback() { + @Override + @SuppressWarnings("unchecked") + public void processResult(int rc, String path, Object ctx) { + SettableOperationFuture result = (SettableOperationFuture) ctx; + KeeperException.Code code = KeeperException.Code.get(rc); + if (code == KeeperException.Code.OK) { + result.set(result.getRequestPath()); + return; + } + // Otherwise, it is an error + result.setException(KeeperException.create(code, result.getRequestPath())); + } + }; + + static final AsyncCallback.ACLCallback ACL = new AsyncCallback.ACLCallback() { + @Override + @SuppressWarnings("unchecked") + public void processResult(int rc, String path, Object ctx, List acl, Stat stat) { + SettableOperationFuture result = (SettableOperationFuture) ctx; + KeeperException.Code code = KeeperException.Code.get(rc); + if (code == KeeperException.Code.OK) { + result.set(new BasicACLData(acl, stat)); + return; + } + result.setException(KeeperException.create(code, result.getRequestPath())); + } + }; + } +} diff --git a/cdap-common/src/main/java/org/apache/twill/internal/zookeeper/FailureRetryZKClient.java b/cdap-common/src/main/java/org/apache/twill/internal/zookeeper/FailureRetryZKClient.java new file mode 100644 index 000000000000..adb1706a3a89 --- /dev/null +++ b/cdap-common/src/main/java/org/apache/twill/internal/zookeeper/FailureRetryZKClient.java @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.twill.internal.zookeeper; + +import com.google.common.base.Supplier; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.MoreExecutors; +import org.apache.twill.common.Threads; +import org.apache.twill.zookeeper.ACLData; +import org.apache.twill.zookeeper.ForwardingZKClient; +import org.apache.twill.zookeeper.NodeChildren; +import org.apache.twill.zookeeper.NodeData; +import org.apache.twill.zookeeper.OperationFuture; +import org.apache.twill.zookeeper.RetryStrategy; +import org.apache.twill.zookeeper.RetryStrategy.OperationType; +import org.apache.twill.zookeeper.ZKClient; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Stat; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import javax.annotation.Nullable; + +/** + * A {@link ZKClient} that will invoke {@link RetryStrategy} on operation failure. + * This {@link ZKClient} works by delegating calls to another {@link ZKClient} + * and listen for the result. If the result is a failure, and is + * {@link RetryUtils#canRetry(org.apache.zookeeper.KeeperException.Code) retryable}, the given {@link RetryStrategy} + * will be called to determine the next retry time, or give up, depending on the value returned by the strategy. + */ +public final class FailureRetryZKClient extends ForwardingZKClient { + + private static final ScheduledExecutorService SCHEDULER = Executors.newSingleThreadScheduledExecutor( + Threads.createDaemonThreadFactory("retry-zkclient")); + private final RetryStrategy retryStrategy; + + public FailureRetryZKClient(ZKClient delegate, RetryStrategy retryStrategy) { + super(delegate); + this.retryStrategy = retryStrategy; + } + + @Override + public OperationFuture create(final String path, @Nullable final byte[] data, final CreateMode createMode, + final boolean createParent, final Iterable acl) { + // No retry for any SEQUENTIAL node, as some algorithms depends on only one sequential node being created. + if (createMode == CreateMode.PERSISTENT_SEQUENTIAL || createMode == CreateMode.EPHEMERAL_SEQUENTIAL) { + return super.create(path, data, createMode, createParent, acl); + } + + final SettableOperationFuture result = SettableOperationFuture.create(path, Threads.SAME_THREAD_EXECUTOR); + Futures.addCallback(super.create(path, data, createMode, createParent, acl), + new OperationFutureCallback(OperationType.CREATE, System.currentTimeMillis(), + path, result, new Supplier>() { + @Override + public OperationFuture get() { + return FailureRetryZKClient.super.create(path, data, createMode, createParent, acl); + } + }), MoreExecutors.directExecutor()); + return result; + } + + @Override + public OperationFuture exists(final String path, final Watcher watcher) { + final SettableOperationFuture result = SettableOperationFuture.create(path, Threads.SAME_THREAD_EXECUTOR); + Futures.addCallback(super.exists(path, watcher), + new OperationFutureCallback(OperationType.EXISTS, System.currentTimeMillis(), + path, result, new Supplier>() { + @Override + public OperationFuture get() { + return FailureRetryZKClient.super.exists(path, watcher); + } + }), MoreExecutors.directExecutor()); + return result; + } + + @Override + public OperationFuture getChildren(final String path, final Watcher watcher) { + final SettableOperationFuture result = SettableOperationFuture.create(path, + Threads.SAME_THREAD_EXECUTOR); + Futures.addCallback(super.getChildren(path, watcher), + new OperationFutureCallback(OperationType.GET_CHILDREN, + System.currentTimeMillis(), path, result, + new Supplier>() { + @Override + public OperationFuture get() { + return FailureRetryZKClient.super.getChildren(path, watcher); + } + }), MoreExecutors.directExecutor()); + return result; + } + + @Override + public OperationFuture getData(final String path, final Watcher watcher) { + final SettableOperationFuture result = SettableOperationFuture.create(path, Threads.SAME_THREAD_EXECUTOR); + Futures.addCallback(super.getData(path, watcher), + new OperationFutureCallback(OperationType.GET_DATA, System.currentTimeMillis(), + path, result, new Supplier>() { + @Override + public OperationFuture get() { + return FailureRetryZKClient.super.getData(path, watcher); + } + }), MoreExecutors.directExecutor()); + return result; + } + + @Override + public OperationFuture setData(final String dataPath, final byte[] data, final int version) { + final SettableOperationFuture result = SettableOperationFuture.create(dataPath, Threads.SAME_THREAD_EXECUTOR); + Futures.addCallback(super.setData(dataPath, data, version), + new OperationFutureCallback(OperationType.SET_DATA, System.currentTimeMillis(), + dataPath, result, new Supplier>() { + @Override + public OperationFuture get() { + return FailureRetryZKClient.super.setData(dataPath, data, version); + } + }), MoreExecutors.directExecutor()); + return result; + } + + @Override + public OperationFuture delete(final String deletePath, final int version) { + final SettableOperationFuture result = SettableOperationFuture.create(deletePath, + Threads.SAME_THREAD_EXECUTOR); + Futures.addCallback(super.delete(deletePath, version), + new OperationFutureCallback(OperationType.DELETE, System.currentTimeMillis(), + deletePath, result, new Supplier> + () { + @Override + public OperationFuture get() { + return FailureRetryZKClient.super.delete(deletePath, version); + } + }), MoreExecutors.directExecutor()); + return result; + } + + @Override + public OperationFuture getACL(final String path) { + final SettableOperationFuture result = SettableOperationFuture.create(path, Threads.SAME_THREAD_EXECUTOR); + Futures.addCallback(super.getACL(path), + new OperationFutureCallback(OperationType.GET_ACL, System.currentTimeMillis(), + path, result, new Supplier>() { + @Override + public OperationFuture get() { + return FailureRetryZKClient.super.getACL(path); + } + }), MoreExecutors.directExecutor()); + return result; + } + + @Override + public OperationFuture setACL(final String path, final Iterable acl, final int version) { + final SettableOperationFuture result = SettableOperationFuture.create(path, Threads.SAME_THREAD_EXECUTOR); + Futures.addCallback(super.setACL(path, acl, version), + new OperationFutureCallback(OperationType.SET_ACL, System.currentTimeMillis(), + path, result, new Supplier>() { + @Override + public OperationFuture get() { + return FailureRetryZKClient.super.setACL(path, acl, version); + } + }), MoreExecutors.directExecutor()); + return result; + } + + /** + * Callback to watch for operation result and trigger retry if necessary. + * @param Type of operation result. + */ + private final class OperationFutureCallback implements FutureCallback { + + private final OperationType type; + private final long startTime; + private final String path; + private final SettableOperationFuture result; + private final Supplier> retryAction; + private final AtomicInteger failureCount; + + private OperationFutureCallback(OperationType type, long startTime, String path, + SettableOperationFuture result, Supplier> retryAction) { + this.type = type; + this.startTime = startTime; + this.path = path; + this.result = result; + this.retryAction = retryAction; + this.failureCount = new AtomicInteger(0); + } + + @Override + public void onSuccess(V result) { + this.result.set(result); + } + + @Override + public void onFailure(Throwable t) { + if (!doRetry(t)) { + result.setException(t); + } + } + + private boolean doRetry(Throwable t) { + if (!RetryUtils.canRetry(t)) { + return false; + } + + // Determine the relay delay + long nextRetry = retryStrategy.nextRetry(failureCount.incrementAndGet(), startTime, type, path); + if (nextRetry < 0) { + return false; + } + + // Schedule the retry. + SCHEDULER.schedule(new Runnable() { + @Override + public void run() { + Futures.addCallback(retryAction.get(), OperationFutureCallback.this, MoreExecutors.directExecutor()); + } + }, nextRetry, TimeUnit.MILLISECONDS); + + return true; + } + } +} diff --git a/cdap-common/src/main/java/org/apache/twill/internal/zookeeper/NamespaceZKClient.java b/cdap-common/src/main/java/org/apache/twill/internal/zookeeper/NamespaceZKClient.java new file mode 100644 index 000000000000..6c41a693d125 --- /dev/null +++ b/cdap-common/src/main/java/org/apache/twill/internal/zookeeper/NamespaceZKClient.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.twill.internal.zookeeper; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import org.apache.twill.common.Cancellable; +import org.apache.twill.common.Threads; +import org.apache.twill.zookeeper.ACLData; +import org.apache.twill.zookeeper.ForwardingZKClient; +import org.apache.twill.zookeeper.NodeChildren; +import org.apache.twill.zookeeper.NodeData; +import org.apache.twill.zookeeper.OperationFuture; +import org.apache.twill.zookeeper.ZKClient; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Stat; + +import javax.annotation.Nullable; + +/** + * A {@link ZKClient} that namespace every paths. + */ +public final class NamespaceZKClient extends ForwardingZKClient { + // This class extends from ForwardingZKClient but overrides every method is for letting the + // ZKClientServices delegate logic works. + + private final String namespace; + private final ZKClient delegate; + private final String connectString; + + public NamespaceZKClient(ZKClient delegate, String namespace) { + super(delegate); + this.namespace = namespace; + this.delegate = delegate; + this.connectString = delegate.getConnectString() + namespace; + } + + @Override + public Long getSessionId() { + return delegate.getSessionId(); + } + + @Override + public String getConnectString() { + return connectString; + } + + @Override + public Cancellable addConnectionWatcher(Watcher watcher) { + return delegate.addConnectionWatcher(watcher); + } + + @Override + public OperationFuture create(String path, @Nullable byte[] data, + CreateMode createMode, boolean createParent, Iterable acl) { + return relayPath(delegate.create(getNamespacedPath(path), data, createMode, createParent, acl), + this.createFuture(path)); + } + + @Override + public OperationFuture exists(String path, @Nullable Watcher watcher) { + return relayFuture(delegate.exists(getNamespacedPath(path), watcher), this.createFuture(path)); + } + + @Override + public OperationFuture getChildren(String path, @Nullable Watcher watcher) { + return relayFuture(delegate.getChildren(getNamespacedPath(path), watcher), this.createFuture(path)); + } + + @Override + public OperationFuture getData(String path, @Nullable Watcher watcher) { + return relayFuture(delegate.getData(getNamespacedPath(path), watcher), this.createFuture(path)); + } + + @Override + public OperationFuture setData(String dataPath, byte[] data, int version) { + return relayFuture(delegate.setData(getNamespacedPath(dataPath), data, version), this.createFuture(dataPath)); + } + + @Override + public OperationFuture delete(String deletePath, int version) { + return relayPath(delegate.delete(getNamespacedPath(deletePath), version), this.createFuture(deletePath)); + } + + @Override + public OperationFuture getACL(String path) { + return relayFuture(delegate.getACL(getNamespacedPath(path)), this.createFuture(path)); + } + + @Override + public OperationFuture setACL(String path, Iterable acl, int version) { + return relayFuture(delegate.setACL(getNamespacedPath(path), acl, version), this.createFuture(path)); + } + + /** + * Returns the namespaced path for the given path. The returned path should be used when performing + * ZK operations with the delegating ZKClient. + */ + private String getNamespacedPath(String path) { + if ("/".equals(path)) { + return namespace; + } + return namespace + path; + } + + private SettableOperationFuture createFuture(String path) { + return SettableOperationFuture.create(path, Threads.SAME_THREAD_EXECUTOR); + } + + private OperationFuture relayFuture(final OperationFuture from, final SettableOperationFuture to) { + Futures.addCallback(from, new FutureCallback() { + @Override + public void onSuccess(V result) { + to.set(result); + } + + @Override + public void onFailure(Throwable t) { + to.setException(t); + } + }, Threads.SAME_THREAD_EXECUTOR); + return to; + } + + private OperationFuture relayPath(final OperationFuture from, + final SettableOperationFuture to) { + from.addListener(new Runnable() { + @Override + public void run() { + try { + String relativePath = from.get().substring(namespace.length()); + to.set(relativePath.isEmpty() ? "/" : relativePath); + } catch (Exception e) { + to.setException(e.getCause()); + } + } + }, Threads.SAME_THREAD_EXECUTOR); + return to; + } +} diff --git a/cdap-common/src/main/java/org/apache/twill/internal/zookeeper/RewatchOnExpireZKClient.java b/cdap-common/src/main/java/org/apache/twill/internal/zookeeper/RewatchOnExpireZKClient.java new file mode 100644 index 000000000000..708cc78daca1 --- /dev/null +++ b/cdap-common/src/main/java/org/apache/twill/internal/zookeeper/RewatchOnExpireZKClient.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.twill.internal.zookeeper; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.MoreExecutors; +import org.apache.twill.internal.zookeeper.RewatchOnExpireWatcher.ActionType; +import org.apache.twill.zookeeper.ForwardingZKClient; +import org.apache.twill.zookeeper.NodeChildren; +import org.apache.twill.zookeeper.NodeData; +import org.apache.twill.zookeeper.OperationFuture; +import org.apache.twill.zookeeper.ZKClient; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.data.Stat; + +/** + * A {@link ZKClient} that will rewatch automatically when session expired and reconnect. + * The rewatch logic is mainly done in {@link RewatchOnExpireWatcher}. + */ +public final class RewatchOnExpireZKClient extends ForwardingZKClient { + + public RewatchOnExpireZKClient(ZKClient delegate) { + super(delegate); + } + + @Override + public OperationFuture exists(String path, Watcher watcher) { + if (watcher == null) { + return super.exists(path, null); + } + final RewatchOnExpireWatcher wrappedWatcher = new RewatchOnExpireWatcher(this, ActionType.EXISTS, path, watcher); + OperationFuture result = super.exists(path, wrappedWatcher); + Futures.addCallback(result, new FutureCallback() { + @Override + public void onSuccess(Stat result) { + wrappedWatcher.setLastResult(result); + } + + @Override + public void onFailure(Throwable t) { + // No-op + } + }, MoreExecutors.directExecutor()); + return result; + } + + @Override + public OperationFuture getChildren(String path, Watcher watcher) { + if (watcher == null) { + return super.getChildren(path, null); + } + final RewatchOnExpireWatcher wrappedWatcher = new RewatchOnExpireWatcher(this, ActionType.CHILDREN, path, watcher); + OperationFuture result = super.getChildren(path, wrappedWatcher); + Futures.addCallback(result, new FutureCallback() { + @Override + public void onSuccess(NodeChildren result) { + wrappedWatcher.setLastResult(result); + } + + @Override + public void onFailure(Throwable t) { + // No-op + } + }, MoreExecutors.directExecutor()); + return result; + } + + @Override + public OperationFuture getData(String path, Watcher watcher) { + if (watcher == null) { + return super.getData(path, null); + } + final RewatchOnExpireWatcher wrappedWatcher = new RewatchOnExpireWatcher(this, ActionType.DATA, path, watcher); + OperationFuture result = super.getData(path, wrappedWatcher); + Futures.addCallback(result, new FutureCallback() { + @Override + public void onSuccess(NodeData result) { + wrappedWatcher.setLastResult(result); + } + + @Override + public void onFailure(Throwable t) { + // No-op + } + }, MoreExecutors.directExecutor()); + return result; + + } +} diff --git a/cdap-common/src/test/java/io/cdap/cdap/common/conf/ZKPropertyStoreTest.java b/cdap-common/src/test/java/io/cdap/cdap/common/conf/ZKPropertyStoreTest.java index 63396168949b..cf29788d1188 100644 --- a/cdap-common/src/test/java/io/cdap/cdap/common/conf/ZKPropertyStoreTest.java +++ b/cdap-common/src/test/java/io/cdap/cdap/common/conf/ZKPropertyStoreTest.java @@ -42,12 +42,12 @@ public static void init() throws IOException { zkServer.startAndWait(); zkClient = ZKClientService.Builder.of(zkServer.getConnectionStr()).build(); - zkClient.startAndWait(); + zkClient.startAsync().awaitRunning(); } @AfterClass public static void finish() { - zkClient.stopAndWait(); + zkClient.stopAsync().awaitTerminated(); zkServer.stopAndWait(); } diff --git a/cdap-common/src/test/java/io/cdap/cdap/common/guice/KafkaClientModuleTest.java b/cdap-common/src/test/java/io/cdap/cdap/common/guice/KafkaClientModuleTest.java index b41436c0b7bb..bcb8a09e5013 100644 --- a/cdap-common/src/test/java/io/cdap/cdap/common/guice/KafkaClientModuleTest.java +++ b/cdap-common/src/test/java/io/cdap/cdap/common/guice/KafkaClientModuleTest.java @@ -71,20 +71,20 @@ public void beforeTest() throws Exception { if (kafkaZkNamespace != null) { ZKClientService zkClient = new DefaultZKClientService(zkServer.getConnectionStr(), 2000, null, ImmutableMultimap.of()); - zkClient.startAndWait(); + zkClient.startAsync().awaitRunning(); zkClient.create("/" + kafkaZkNamespace, null, CreateMode.PERSISTENT); - zkClient.stopAndWait(); + zkClient.stopAsync().awaitTerminated(); kafkaZkConnect += "/" + kafkaZkNamespace; } kafkaServer = createKafkaServer(kafkaZkConnect, TEMP_FOLDER.newFolder()); - kafkaServer.startAndWait(); + kafkaServer.startAsync().awaitRunning(); } @After public void afterTest() { - kafkaServer.stopAndWait(); + kafkaServer.stopAsync().awaitTerminated(); zkServer.stopAndWait(); } @@ -101,7 +101,7 @@ public void testWithSharedZkClient() throws Exception { // Get the shared zkclient and start it ZKClientService zkClientService = injector.getInstance(ZKClientService.class); - zkClientService.startAndWait(); + zkClientService.startAsync().awaitRunning(); final int baseZkConns = getZkConnections(); @@ -109,8 +109,8 @@ public void testWithSharedZkClient() throws Exception { final BrokerService brokerService = injector.getInstance(BrokerService.class); // Start both kafka and broker services, it shouldn't affect the state of the shared zk client - kafkaClientService.startAndWait(); - brokerService.startAndWait(); + kafkaClientService.startAsync().awaitRunning(); + brokerService.startAsync().awaitRunning(); // Shouldn't affect the shared zk client state Assert.assertTrue(zkClientService.isRunning()); @@ -127,8 +127,8 @@ public Boolean call() throws Exception { }, 5L, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); // Stop both, still shouldn't affect the state of the shared zk client - kafkaClientService.stopAndWait(); - brokerService.stopAndWait(); + kafkaClientService.stopAsync().awaitTerminated(); + brokerService.stopAsync().awaitTerminated(); // Still shouldn't affect the shared zk client Assert.assertTrue(zkClientService.isRunning()); @@ -136,7 +136,7 @@ public Boolean call() throws Exception { // It still shouldn't increase the number of zk client connections Assert.assertEquals(baseZkConns, getZkConnections()); - zkClientService.stopAndWait(); + zkClientService.stopAsync().awaitTerminated(); } @Test @@ -154,7 +154,7 @@ public void testWithDedicatedZkClient() throws Exception { // Get the shared zkclient and start it ZKClientService zkClientService = injector.getInstance(ZKClientService.class); - zkClientService.startAndWait(); + zkClientService.startAsync().awaitRunning(); int baseZkConns = getZkConnections(); @@ -162,12 +162,12 @@ public void testWithDedicatedZkClient() throws Exception { final BrokerService brokerService = injector.getInstance(BrokerService.class); // Start the kafka client, it should increase the zk connections by 1 - kafkaClientService.startAndWait(); + kafkaClientService.startAsync().awaitRunning(); Assert.assertEquals(baseZkConns + 1, getZkConnections()); // Start the broker service, // it shouldn't affect the zk connections, as it share the same zk client with kafka client - brokerService.startAndWait(); + brokerService.startAsync().awaitRunning(); Assert.assertEquals(baseZkConns + 1, getZkConnections()); // Make sure it is talking to Kafka. @@ -182,17 +182,17 @@ public Boolean call() throws Exception { Assert.assertTrue(zkClientService.isRunning()); // Stop the broker service, it shouldn't affect the zk connections, as it is still used by the kafka client - brokerService.stopAndWait(); + brokerService.stopAsync().awaitTerminated(); Assert.assertEquals(baseZkConns + 1, getZkConnections()); // Stop the kafka client, the zk connections should be reduced by 1 - kafkaClientService.stopAndWait(); + kafkaClientService.stopAsync().awaitTerminated(); Assert.assertEquals(baseZkConns, getZkConnections()); // Still shouldn't affect the shared zk client Assert.assertTrue(zkClientService.isRunning()); - zkClientService.stopAndWait(); + zkClientService.stopAsync().awaitTerminated(); } /** diff --git a/cdap-common/src/test/java/io/cdap/cdap/common/guice/ZkDiscoveryModuleTest.java b/cdap-common/src/test/java/io/cdap/cdap/common/guice/ZkDiscoveryModuleTest.java index c809a29c2824..d106dd5d5211 100644 --- a/cdap-common/src/test/java/io/cdap/cdap/common/guice/ZkDiscoveryModuleTest.java +++ b/cdap-common/src/test/java/io/cdap/cdap/common/guice/ZkDiscoveryModuleTest.java @@ -80,7 +80,7 @@ public void testMasterDiscovery() { ); ZKClientService zkClient = injector.getInstance(ZKClientService.class); - zkClient.startAndWait(); + zkClient.startAsync().awaitRunning(); try { DiscoveryService discoveryService = injector.getInstance(DiscoveryService.class); DiscoveryServiceClient discoveryServiceClient = injector @@ -106,7 +106,7 @@ public void testMasterDiscovery() { } } finally { - zkClient.stopAndWait(); + zkClient.stopAsync().awaitTerminated(); } } @@ -119,7 +119,7 @@ public void testProgramDiscovery() { ); ZKClientService zkClient = injector.getInstance(ZKClientService.class); - zkClient.startAndWait(); + zkClient.startAsync().awaitRunning(); try { // Register a service using the twill ZKClient. This is to simulate how a user Service program register ProgramId programId = NamespaceId.DEFAULT.app("app").service("service"); @@ -149,7 +149,7 @@ public void testProgramDiscovery() { } } } finally { - zkClient.stopAndWait(); + zkClient.stopAsync().awaitTerminated(); } } } diff --git a/cdap-common/src/test/java/io/cdap/cdap/common/internal/remote/RemoteTaskExecutorTest.java b/cdap-common/src/test/java/io/cdap/cdap/common/internal/remote/RemoteTaskExecutorTest.java index 5781de705674..024058a58dc2 100644 --- a/cdap-common/src/test/java/io/cdap/cdap/common/internal/remote/RemoteTaskExecutorTest.java +++ b/cdap-common/src/test/java/io/cdap/cdap/common/internal/remote/RemoteTaskExecutorTest.java @@ -90,7 +90,7 @@ public void modify(ChannelPipeline pipeline) { public void beforeTest() { metricCollectors = new HashMap<>(); mockMetricsCollector = createMockMetricsCollectionService(); - mockMetricsCollector.startAndWait(); + mockMetricsCollector.startAsync().awaitRunning(); registered = discoveryService.register(URIScheme.createDiscoverable(Constants.Service.TASK_WORKER, httpService)); } @@ -112,37 +112,54 @@ private MetricsCollectionService createMockMetricsCollectionService() { return new MetricsCollectionService() { @Override - public ListenableFuture start() { - return null; + public com.google.common.util.concurrent.Service startAsync() { + return this; } @Override - public State startAndWait() { - return null; + public com.google.common.util.concurrent.Service stopAsync() { + return this; } @Override - public boolean isRunning() { - return false; + public void awaitRunning() { + // no-op } @Override - public State state() { - return null; + public void awaitRunning(long timeout, java.util.concurrent.TimeUnit unit) + throws java.util.concurrent.TimeoutException { + // no-op } @Override - public ListenableFuture stop() { - return null; + public void awaitTerminated() { + // no-op + } + + @Override + public void awaitTerminated(long timeout, java.util.concurrent.TimeUnit unit) + throws java.util.concurrent.TimeoutException { + // no-op } @Override - public State stopAndWait() { + public Throwable failureCause() { return null; } @Override - public void addListener(final Listener listener, final Executor executor) {} + public boolean isRunning() { + return true; + } + + @Override + public State state() { + return State.RUNNING; + } + + @Override + public void addListener(Listener listener, Executor executor) {} @Override public MetricsContext getContext(Map context) { @@ -205,7 +222,7 @@ public void testFailedMetrics() throws Exception { // Exception thrown in the task executor should be in the exception message in the caller Assert.assertEquals("Invalid", e.getMessage()); } - mockMetricsCollector.stopAndWait(); + mockMetricsCollector.stopAsync().awaitTerminated(); Assert.assertSame(1, metricCollectors.size()); //check the metrics are present @@ -224,7 +241,7 @@ public void testSuccessMetrics() throws Exception { RunnableTaskRequest runnableTaskRequest = RunnableTaskRequest.getBuilder(ValidRunnableClass.class.getName()). withParam("param").withNamespace("testNamespace").build(); remoteTaskExecutor.runTask(runnableTaskRequest); - mockMetricsCollector.stopAndWait(); + mockMetricsCollector.stopAsync().awaitTerminated(); Assert.assertSame(1, metricCollectors.size()); //check the metrics are present @@ -249,7 +266,7 @@ public void testRetryMetrics() throws Exception { } catch (Exception e) { // expected } - mockMetricsCollector.stopAndWait(); + mockMetricsCollector.stopAsync().awaitTerminated(); Assert.assertSame(1, metricCollectors.size()); //check the metrics are present diff --git a/cdap-common/src/test/java/io/cdap/cdap/common/resource/ResourceBalancerServiceTest.java b/cdap-common/src/test/java/io/cdap/cdap/common/resource/ResourceBalancerServiceTest.java index 8a5d4b815491..9cd1fc64ad43 100644 --- a/cdap-common/src/test/java/io/cdap/cdap/common/resource/ResourceBalancerServiceTest.java +++ b/cdap-common/src/test/java/io/cdap/cdap/common/resource/ResourceBalancerServiceTest.java @@ -72,13 +72,13 @@ public void testResourceBalancerService() throws Exception { // Simple test for resource balancer does react to discovery changes correct // More detailed tests are in ResourceCoordinatorTest, which the ResourceBalancerService depends on ZKClientService zkClient = ZKClientService.Builder.of(zkServer.getConnectionStr()).build(); - zkClient.startAndWait(); + zkClient.startAsync().awaitRunning(); try (ZKDiscoveryService discoveryService = new ZKDiscoveryService(zkClient)) { // Test the failure on stop case final TestBalancerService stopFailureService = new TestBalancerService("test", 4, zkClient, discoveryService, discoveryService, false, false); - stopFailureService.startAndWait(); + stopFailureService.startAsync().awaitRunning(); // Should get all four partitions Tasks.waitFor(ImmutableSet.of(0, 1, 2, 3), new Callable>() { @@ -103,20 +103,20 @@ public Integer call() throws Exception { cancellable.cancel(); } } finally { - zkClient.stopAndWait(); + zkClient.stopAsync().awaitTerminated(); } } @Test public void testServiceStartFailure() throws Exception { ZKClientService zkClient = ZKClientService.Builder.of(zkServer.getConnectionStr()).build(); - zkClient.startAndWait(); + zkClient.startAsync().awaitRunning(); try (ZKDiscoveryService discoveryService = new ZKDiscoveryService(zkClient)) { // Test the failure on start case final TestBalancerService startFailureService = new TestBalancerService("test", 4, zkClient, discoveryService, discoveryService, true, false); - startFailureService.startAndWait(); + startFailureService.startAsync().awaitRunning(); // The resource balance service should fail Tasks.waitFor(Service.State.FAILED, new Callable() { @@ -126,20 +126,20 @@ public Service.State call() throws Exception { } }, 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); } finally { - zkClient.stopAndWait(); + zkClient.stopAsync().awaitTerminated(); } } @Test public void testServiceStopFailure() throws Exception { ZKClientService zkClient = ZKClientService.Builder.of(zkServer.getConnectionStr()).build(); - zkClient.startAndWait(); + zkClient.startAsync().awaitRunning(); try (ZKDiscoveryService discoveryService = new ZKDiscoveryService(zkClient)) { // Test the failure on stop case final TestBalancerService stopFailureService = new TestBalancerService("test", 4, zkClient, discoveryService, discoveryService, false, true); - stopFailureService.startAndWait(); + stopFailureService.startAsync().awaitRunning(); // Should get four partitions Tasks.waitFor(ImmutableSet.of(0, 1, 2, 3), new Callable>() { @@ -165,7 +165,7 @@ public Service.State call() throws Exception { cancellable.cancel(); } } finally { - zkClient.stopAndWait(); + zkClient.stopAsync().awaitTerminated(); } } diff --git a/cdap-common/src/test/java/io/cdap/cdap/common/service/CommandPortServiceTest.java b/cdap-common/src/test/java/io/cdap/cdap/common/service/CommandPortServiceTest.java index 6aee3250ddb2..35281e269f20 100644 --- a/cdap-common/src/test/java/io/cdap/cdap/common/service/CommandPortServiceTest.java +++ b/cdap-common/src/test/java/io/cdap/cdap/common/service/CommandPortServiceTest.java @@ -16,8 +16,7 @@ package io.cdap.cdap.common.service; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.Service; import java.io.BufferedReader; import java.io.BufferedWriter; @@ -62,17 +61,18 @@ public void testCommandPortServer() throws Exception { .build(); final CountDownLatch stopLatch = new CountDownLatch(1); - Futures.addCallback(server.start(), new FutureCallback() { + server.addListener(new Service.Listener() { @Override - public void onSuccess(Service.State result) { + public void running() { stopLatch.countDown(); } @Override - public void onFailure(Throwable t) { + public void failed(Service.State from, Throwable failure) { stopLatch.countDown(); } - }); + }, MoreExecutors.directExecutor()); + server.startAsync(); // wait a bit for service to start TimeUnit.SECONDS.sleep(1); @@ -95,7 +95,7 @@ public void onFailure(Throwable t) { } } finally { - server.stopAndWait(); + server.stopAsync().awaitTerminated(); } Assert.assertEquals(10, handler.getCounter()); diff --git a/cdap-common/src/test/java/io/cdap/cdap/common/service/RetryOnStartFailureServiceTest.java b/cdap-common/src/test/java/io/cdap/cdap/common/service/RetryOnStartFailureServiceTest.java index b2233dc07704..5d641bfec652 100644 --- a/cdap-common/src/test/java/io/cdap/cdap/common/service/RetryOnStartFailureServiceTest.java +++ b/cdap-common/src/test/java/io/cdap/cdap/common/service/RetryOnStartFailureServiceTest.java @@ -42,7 +42,7 @@ public void testRetrySucceed() throws InterruptedException { Service service = new RetryOnStartFailureService( createServiceSupplier(3, startLatch, new CountDownLatch(1), false), RetryStrategies.fixDelay(10, TimeUnit.MILLISECONDS)); - service.startAndWait(); + service.startAsync().awaitRunning(); Assert.assertTrue(startLatch.await(1, TimeUnit.SECONDS)); } @@ -54,14 +54,14 @@ public void testRetryFail() throws InterruptedException { RetryStrategies.limit(10, RetryStrategies.fixDelay(10, TimeUnit.MILLISECONDS))); final CountDownLatch failureLatch = new CountDownLatch(1); - service.addListener(new ServiceListenerAdapter() { + service.addListener(new Service.Listener() { @Override public void failed(Service.State from, Throwable failure) { failureLatch.countDown(); } }, Threads.SAME_THREAD_EXECUTOR); - service.start(); + service.startAsync(); Assert.assertTrue(failureLatch.await(1, TimeUnit.SECONDS)); Assert.assertFalse(startLatch.await(100, TimeUnit.MILLISECONDS)); } @@ -73,9 +73,9 @@ public void testStopWhileRetrying() throws InterruptedException { Service service = new RetryOnStartFailureService( createServiceSupplier(1000, new CountDownLatch(1), failureLatch, false), RetryStrategies.fixDelay(10, TimeUnit.MILLISECONDS)); - service.startAndWait(); + service.startAsync().awaitRunning(); Assert.assertTrue(failureLatch.await(1, TimeUnit.SECONDS)); - service.stopAndWait(); + service.stopAsync().awaitTerminated(); } @Test @@ -85,7 +85,7 @@ public void testStopFailurePropagate() throws InterruptedException, TimeoutExcep final RetryOnStartFailureService service = new RetryOnStartFailureService( createServiceSupplier(0, startLatch, new CountDownLatch(1), true), RetryStrategies.fixDelay(10, TimeUnit.MILLISECONDS)); - service.startAndWait(); + service.startAsync().awaitRunning(); // block until the underlying service started successfully Assert.assertTrue(startLatch.await(1, TimeUnit.SECONDS)); // As documented in the RetryOnStartFailureService, there is a small race after the @@ -99,7 +99,7 @@ public Boolean call() throws Exception { } }, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); try { - service.stopAndWait(); + service.stopAsync().awaitTerminated(); Assert.fail("Expected failure in stopping"); } catch (Exception e) { Assert.assertEquals("Intentional failure to shutdown", Throwables.getRootCause(e).getMessage()); diff --git a/cdap-common/src/test/java/io/cdap/cdap/common/service/RetryableScheduledServiceTest.java b/cdap-common/src/test/java/io/cdap/cdap/common/service/RetryableScheduledServiceTest.java index 2ce4ce226215..6fe1bb793b01 100644 --- a/cdap-common/src/test/java/io/cdap/cdap/common/service/RetryableScheduledServiceTest.java +++ b/cdap-common/src/test/java/io/cdap/cdap/common/service/RetryableScheduledServiceTest.java @@ -45,9 +45,9 @@ protected long runTask() { } }; - service.start(); + service.startAsync(); Assert.assertTrue(latch.await(5, TimeUnit.SECONDS)); - service.stopAndWait(); + service.stopAsync().awaitTerminated(); } @Test @@ -59,12 +59,12 @@ protected long runTask() throws Exception { } }; - service.start(); + service.startAsync(); // Wait for the service to fail Tasks.waitFor(Service.State.FAILED, service::state, 5, TimeUnit.SECONDS, 10, TimeUnit.MILLISECONDS); try { - service.stopAndWait(); + service.stopAsync().awaitTerminated(); } catch (Exception e) { // The root cause should be the one throw from the runTask. It should suppressed the retry exhausted exception. Throwable rootCause = Throwables.getRootCause(e); @@ -89,12 +89,12 @@ protected boolean shouldRetry(Exception ex) { } }; - service.start(); + service.startAsync(); // Wait for the service to fail Tasks.waitFor(Service.State.FAILED, service::state, 5, TimeUnit.SECONDS, 10, TimeUnit.MILLISECONDS); try { - service.stopAndWait(); + service.stopAsync().awaitTerminated(); } catch (Exception e) { // The root cause should be the one throw from the runTask. Throwable rootCause = Throwables.getRootCause(e); @@ -118,8 +118,8 @@ protected long runTask() throws Exception { return 1L; } }; - service.start(); + service.startAsync(); Assert.assertTrue(latch.await(3, TimeUnit.SECONDS)); - service.stopAndWait(); + service.stopAsync().awaitTerminated(); } } diff --git a/cdap-common/src/test/java/io/cdap/cdap/common/ssh/SSHSessionTest.java b/cdap-common/src/test/java/io/cdap/cdap/common/ssh/SSHSessionTest.java index 8fcf8a0014b8..c40d40135559 100644 --- a/cdap-common/src/test/java/io/cdap/cdap/common/ssh/SSHSessionTest.java +++ b/cdap-common/src/test/java/io/cdap/cdap/common/ssh/SSHSessionTest.java @@ -131,7 +131,7 @@ public void testLocalPortForwarding() throws Exception { // Starts an echo server for testing the port forwarding EchoServer echoServer = new EchoServer(); - echoServer.startAndWait(); + echoServer.startAsync().awaitRunning(); try { // Creates the DataConsumer for receiving data and validating the lifecycle StringBuilder received = new StringBuilder(); @@ -185,7 +185,7 @@ public void finished() { } } finally { - echoServer.stopAndWait(); + echoServer.stopAsync().awaitTerminated(); } } @@ -193,7 +193,7 @@ public void finished() { public void testForwardingOnSessionClose() throws Exception { EchoServer echoServer = new EchoServer(); - echoServer.startAndWait(); + echoServer.startAsync().awaitRunning(); try { SSHConfig sshConfig = getSSHConfig(); AtomicBoolean finished = new AtomicBoolean(false); @@ -236,7 +236,7 @@ public void finished() { } } finally { - echoServer.stopAndWait(); + echoServer.stopAsync().awaitTerminated(); } } @@ -244,7 +244,7 @@ public void finished() { public void testRemotePortForwarding() throws Exception { EchoServer echoServer = new EchoServer(); - echoServer.startAndWait(); + echoServer.startAsync().awaitRunning(); try { SSHConfig sshConfig = getSSHConfig(); @@ -264,7 +264,7 @@ public void testRemotePortForwarding() throws Exception { } } } finally { - echoServer.stopAndWait(); + echoServer.stopAsync().awaitTerminated(); } } @@ -320,7 +320,13 @@ protected void run() throws IOException { } catch (IOException e) { LOG.error("Exception raised from the EchoServer handling thread", e); } finally { - Closeables.closeQuietly(socket); + try { + if (socket != null) { + socket.close(); + } + } catch (IOException e) { + // Ignore + } } }); @@ -337,7 +343,13 @@ protected void run() throws IOException { @Override protected void triggerShutdown() { stopped = true; - Closeables.closeQuietly(serverSocket); + try { + if (serverSocket != null) { + serverSocket.close(); + } + } catch (IOException e) { + // Ignore + } } } } diff --git a/cdap-common/src/test/java/io/cdap/cdap/common/utils/TimeBoundIteratorTest.java b/cdap-common/src/test/java/io/cdap/cdap/common/utils/TimeBoundIteratorTest.java index 34758d971a5a..84635401c982 100644 --- a/cdap-common/src/test/java/io/cdap/cdap/common/utils/TimeBoundIteratorTest.java +++ b/cdap-common/src/test/java/io/cdap/cdap/common/utils/TimeBoundIteratorTest.java @@ -34,7 +34,7 @@ public class TimeBoundIteratorTest { @Test public void testTimeBoundNotHit() { SettableTicker ticker = new SettableTicker(0); - Stopwatch stopwatch = new Stopwatch(ticker); + Stopwatch stopwatch = Stopwatch.createUnstarted(ticker); List list = new ArrayList<>(); list.add(0); @@ -54,7 +54,7 @@ public void testTimeBoundNotHit() { @Test public void testTimeBoundImmediatelyHit() { SettableTicker ticker = new SettableTicker(0); - Stopwatch stopwatch = new Stopwatch(ticker); + Stopwatch stopwatch = Stopwatch.createUnstarted(ticker); List list = new ArrayList<>(); list.add(0); @@ -70,7 +70,7 @@ public void testTimeBoundImmediatelyHit() { @Test public void testEarlyStop() { SettableTicker ticker = new SettableTicker(0); - Stopwatch stopwatch = new Stopwatch(ticker); + Stopwatch stopwatch = Stopwatch.createUnstarted(ticker); List list = new ArrayList<>(); list.add(0); diff --git a/cdap-common/src/test/java/io/cdap/cdap/common/zookeeper/ZKExtOperationsTest.java b/cdap-common/src/test/java/io/cdap/cdap/common/zookeeper/ZKExtOperationsTest.java index 4abd581060df..dcc603879ddf 100644 --- a/cdap-common/src/test/java/io/cdap/cdap/common/zookeeper/ZKExtOperationsTest.java +++ b/cdap-common/src/test/java/io/cdap/cdap/common/zookeeper/ZKExtOperationsTest.java @@ -68,8 +68,8 @@ public void testGetAndSet() throws Exception { ZKClientService zkClient1 = ZKClientService.Builder.of(zkServer.getConnectionStr()).build(); ZKClientService zkClient2 = ZKClientService.Builder.of(zkServer.getConnectionStr()).build(); - zkClient1.startAndWait(); - zkClient2.startAndWait(); + zkClient1.startAsync().awaitRunning(); + zkClient2.startAsync().awaitRunning(); // First a node would get created since no node there. ZKExtOperations.updateOrCreate(zkClient1, path, new Function() { @@ -134,15 +134,15 @@ public Integer apply(@Nullable Integer input) { Assert.assertNull(result); - zkClient1.stopAndWait(); - zkClient2.stopAndWait(); + zkClient1.stopAsync().awaitTerminated(); + zkClient2.stopAsync().awaitTerminated(); } @Test public void testCreateOrSet() throws Exception { String path = "/parent/testCreateOrSet"; ZKClientService zkClient = ZKClientService.Builder.of(zkServer.getConnectionStr()).build(); - zkClient.startAndWait(); + zkClient.startAsync().awaitRunning(); // Create with "1" Assert.assertEquals(1, ZKExtOperations.createOrSet(zkClient, path, @@ -156,14 +156,14 @@ public void testCreateOrSet() throws Exception { // Should get "2" back Assert.assertEquals(2, INT_CODEC.decode(zkClient.getData(path).get().getData()).intValue()); - zkClient.stopAndWait(); + zkClient.stopAsync().awaitTerminated(); } @Test public void testSetOrCreate() throws Exception { String path = "/parent/testSetOrCreate"; ZKClientService zkClient = ZKClientService.Builder.of(zkServer.getConnectionStr()).build(); - zkClient.startAndWait(); + zkClient.startAsync().awaitRunning(); // Create with "1" Assert.assertEquals(1, ZKExtOperations.setOrCreate(zkClient, path, @@ -177,7 +177,7 @@ public void testSetOrCreate() throws Exception { // Should get "2" back Assert.assertEquals(2, INT_CODEC.decode(zkClient.getData(path).get().getData()).intValue()); - zkClient.stopAndWait(); + zkClient.stopAsync().awaitTerminated(); } @AfterClass diff --git a/cdap-common/src/test/java/io/cdap/cdap/common/zookeeper/coordination/ResourceCoordinatorTest.java b/cdap-common/src/test/java/io/cdap/cdap/common/zookeeper/coordination/ResourceCoordinatorTest.java index b7813fb679bc..2e1e78d74c80 100644 --- a/cdap-common/src/test/java/io/cdap/cdap/common/zookeeper/coordination/ResourceCoordinatorTest.java +++ b/cdap-common/src/test/java/io/cdap/cdap/common/zookeeper/coordination/ResourceCoordinatorTest.java @@ -71,18 +71,18 @@ public void testAssignment() throws InterruptedException, ExecutionException { new ZkClientModule(), new ZkDiscoveryModule()); ZKClientService zkClient = injector.getInstance(ZKClientService.class); - zkClient.startAndWait(); + zkClient.startAsync().awaitRunning(); DiscoveryService discoveryService = injector.getInstance(DiscoveryService.class); try { ResourceCoordinator coordinator = new ResourceCoordinator(zkClient, injector.getInstance(DiscoveryServiceClient.class), new BalancedAssignmentStrategy()); - coordinator.startAndWait(); + coordinator.startAsync().awaitRunning(); try { ResourceCoordinatorClient client = new ResourceCoordinatorClient(zkClient); - client.startAndWait(); + client.startAsync().awaitRunning(); try { // Create a requirement @@ -171,14 +171,14 @@ public void testAssignment() throws InterruptedException, ExecutionException { cancelDiscoverable2.cancel(); } finally { - client.stopAndWait(); + client.stopAsync().awaitTerminated(); } } finally { - coordinator.stopAndWait(); + coordinator.stopAsync().awaitTerminated(); } } finally { - zkClient.stopAndWait(); + zkClient.stopAsync().awaitTerminated(); } } diff --git a/cdap-common/src/test/java/io/cdap/cdap/common/zookeeper/election/LeaderElectionInfoServiceTest.java b/cdap-common/src/test/java/io/cdap/cdap/common/zookeeper/election/LeaderElectionInfoServiceTest.java index 7ce62d05dbb2..e1c84f00bb59 100644 --- a/cdap-common/src/test/java/io/cdap/cdap/common/zookeeper/election/LeaderElectionInfoServiceTest.java +++ b/cdap-common/src/test/java/io/cdap/cdap/common/zookeeper/election/LeaderElectionInfoServiceTest.java @@ -71,12 +71,12 @@ public void testParticipants() throws Exception { List zkClients = new ArrayList<>(); ZKClientService infoZKClient = DefaultZKClientService.Builder.of(zkServer.getConnectionStr()).build(); - infoZKClient.startAndWait(); + infoZKClient.startAsync().awaitRunning(); zkClients.add(infoZKClient); // Start the LeaderElectionInfoService LeaderElectionInfoService infoService = new LeaderElectionInfoService(infoZKClient, prefix); - infoService.startAndWait(); + infoService.startAsync().awaitRunning(); // This will timeout as there is no leader election node created yet try { @@ -90,7 +90,7 @@ public void testParticipants() throws Exception { List leaderElections = new ArrayList<>(); for (int i = 0; i < size; i++) { ZKClientService zkClient = DefaultZKClientService.Builder.of(zkServer.getConnectionStr()).build(); - zkClient.startAndWait(); + zkClient.startAsync().awaitRunning(); zkClients.add(zkClient); final int participantId = i; @@ -105,7 +105,7 @@ public void follower() { LOG.info("Follow: {}", participantId); } }); - leaderElection.start(); + leaderElection.startAsync(); leaderElections.add(leaderElection); } @@ -136,7 +136,7 @@ public boolean apply(LeaderElectionInfoService.Participant input) { int expectedSize = size; for (LeaderElection leaderElection : leaderElections) { - leaderElection.stopAndWait(); + leaderElection.stopAsync().awaitTerminated(); Tasks.waitFor(--expectedSize, new Callable() { @Override public Integer call() throws Exception { @@ -150,10 +150,10 @@ public Integer call() throws Exception { Assert.assertTrue(snapshot.isEmpty()); Assert.assertEquals(participants, snapshot); - infoService.stopAndWait(); + infoService.stopAsync().awaitTerminated(); for (ZKClientService zkClient : zkClients) { - zkClient.stopAndWait(); + zkClient.stopAsync().awaitTerminated(); } } } diff --git a/cdap-credential-ext-gcp-wi/pom.xml b/cdap-credential-ext-gcp-wi/pom.xml index 8bfa8b197ad1..d562bc8d363b 100644 --- a/cdap-credential-ext-gcp-wi/pom.xml +++ b/cdap-credential-ext-gcp-wi/pom.xml @@ -117,6 +117,20 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + dist @@ -148,7 +162,6 @@ org.apache.maven.plugins maven-jar-plugin - 2.4 jar @@ -156,6 +169,7 @@ ${project.build.directory}/libexec ${project.groupId}.${project.build.finalName} + false jar diff --git a/cdap-data-fabric-tests/pom.xml b/cdap-data-fabric-tests/pom.xml index 4ee3d7fd5662..4e06843fcf31 100644 --- a/cdap-data-fabric-tests/pom.xml +++ b/cdap-data-fabric-tests/pom.xml @@ -129,6 +129,15 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + org.apache.maven.plugins maven-surefire-plugin diff --git a/cdap-data-fabric-tests/src/test/java/io/cdap/cdap/data2/transaction/distributed/TransactionServiceClientTest.java b/cdap-data-fabric-tests/src/test/java/io/cdap/cdap/data2/transaction/distributed/TransactionServiceClientTest.java index 72884ad277c8..fa7ab83755e3 100644 --- a/cdap-data-fabric-tests/src/test/java/io/cdap/cdap/data2/transaction/distributed/TransactionServiceClientTest.java +++ b/cdap-data-fabric-tests/src/test/java/io/cdap/cdap/data2/transaction/distributed/TransactionServiceClientTest.java @@ -121,7 +121,7 @@ public static void beforeClass() throws Exception { server = TransactionServiceTest .createTxService(zkServer.getConnectionStr(), Networks.getRandomPort(), hConf, tmpFolder.newFolder(), cConf); - server.startAndWait(); + server.startAsync().awaitRunning(); injector = Guice.createInjector( new ConfigModule(cConf, hConf), @@ -150,25 +150,25 @@ protected void configure() { new AuthenticationContextModules().getNoOpModule()); zkClient = injector.getInstance(ZKClientService.class); - zkClient.startAndWait(); + zkClient.startAsync().awaitRunning(); txStateStorage = injector.getInstance(TransactionStateStorage.class); - txStateStorage.startAndWait(); + txStateStorage.startAsync().awaitRunning(); } @AfterClass public static void afterClass() { try { try { - server.stopAndWait(); + server.stopAsync().awaitTerminated(); miniDfsCluster.shutdown(); } finally { - zkClient.stopAndWait(); - txStateStorage.stopAndWait(); + zkClient.stopAsync().awaitTerminated(); + txStateStorage.stopAsync().awaitTerminated(); } } finally { zkServer.stopAndWait(); - txStateStorage.stopAndWait(); + txStateStorage.stopAsync().awaitTerminated(); } } diff --git a/cdap-data-fabric-tests/src/test/java/io/cdap/cdap/data2/transaction/distributed/TransactionServiceTest.java b/cdap-data-fabric-tests/src/test/java/io/cdap/cdap/data2/transaction/distributed/TransactionServiceTest.java index 0f3d4ea8b623..e9b210b6047e 100644 --- a/cdap-data-fabric-tests/src/test/java/io/cdap/cdap/data2/transaction/distributed/TransactionServiceTest.java +++ b/cdap-data-fabric-tests/src/test/java/io/cdap/cdap/data2/transaction/distributed/TransactionServiceTest.java @@ -146,7 +146,7 @@ protected void configure() { ); ZKClientService zkClient = injector.getInstance(ZKClientService.class); - zkClient.startAndWait(); + zkClient.startAsync().awaitRunning(); try { final Table table = createTable("myTable"); @@ -161,7 +161,7 @@ protected void configure() { TransactionService first = createTxService(zkServer.getConnectionStr(), Networks.getRandomPort(), hConf, tmpFolder.newFolder()); - first.startAndWait(); + first.startAsync().awaitRunning(); Assert.assertNotNull(txClient.startShort()); verifyGetAndPut(table, txExecutor, null, "val1"); @@ -171,7 +171,7 @@ protected void configure() { hConf, tmpFolder.newFolder()); // NOTE: we don't have to wait for start as client should pick it up anyways, but we do wait to ensure // the case with two active is handled well - second.startAndWait(); + second.startAsync().awaitRunning(); // wait for affect a bit TimeUnit.SECONDS.sleep(1); @@ -179,7 +179,7 @@ protected void configure() { verifyGetAndPut(table, txExecutor, "val1", "val2"); // shutting down the first one is fine: we have another one to pick up the leader role - first.stopAndWait(); + first.stopAsync().awaitTerminated(); Assert.assertNotNull(txClient.startShort()); verifyGetAndPut(table, txExecutor, "val2", "val3"); @@ -189,21 +189,21 @@ protected void configure() { Networks.getRandomPort(), hConf, tmpFolder.newFolder()); // NOTE: we don't have to wait for start as client should pick it up anyways - third.start(); + third.startAsync(); // stopping second one - second.stopAndWait(); + second.stopAsync().awaitTerminated(); Assert.assertNotNull(txClient.startShort()); verifyGetAndPut(table, txExecutor, "val3", "val4"); // releasing resources - third.stop(); + third.stopAsync(); } finally { try { dropTable("myTable"); } finally { - zkClient.stopAndWait(); + zkClient.stopAsync().awaitTerminated(); } } } @@ -268,7 +268,7 @@ protected void configure() { new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule()); - injector.getInstance(ZKClientService.class).startAndWait(); + injector.getInstance(ZKClientService.class).startAsync().awaitRunning(); return injector.getInstance(TransactionService.class); } diff --git a/cdap-data-fabric/pom.xml b/cdap-data-fabric/pom.xml index 4b05f35282ee..ee6b9209606c 100644 --- a/cdap-data-fabric/pom.xml +++ b/cdap-data-fabric/pom.xml @@ -195,10 +195,18 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 17 + 17 + + org.apache.maven.plugins maven-jar-plugin - 2.4 test-jar diff --git a/cdap-data-fabric/src/main/java/co/cask/tephra/persist/TransactionEdit.java b/cdap-data-fabric/src/main/java/co/cask/tephra/persist/TransactionEdit.java new file mode 100644 index 000000000000..8d610fd822bf --- /dev/null +++ b/cdap-data-fabric/src/main/java/co/cask/tephra/persist/TransactionEdit.java @@ -0,0 +1,365 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package co.cask.tephra.persist; + +import com.google.common.base.MoreObjects; +import com.google.common.base.Objects; +import com.google.common.collect.Sets; +import org.apache.hadoop.io.Writable; +import org.apache.tephra.ChangeId; +import org.apache.tephra.TransactionManager; +import org.apache.tephra.TransactionType; +import org.apache.tephra.persist.TransactionLog; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Set; + +/** + * Represents a transaction state change in the {@link TransactionLog}. + * This class was included for backward compatibility reasons. It will be removed in future releases. + */ +@Deprecated +public class TransactionEdit implements Writable { + + /** + * The possible state changes for a transaction. + */ + public enum State { + INPROGRESS, COMMITTING, COMMITTED, INVALID, ABORTED, MOVE_WATERMARK, TRUNCATE_INVALID_TX, CHECKPOINT + } + + private long writePointer; + + /** + * stores the value of visibility upper bound + * (see {@link TransactionManager.InProgressTx#getVisibilityUpperBound()}) + * for edit of {@link State#INPROGRESS} only + */ + private long visibilityUpperBound; + private long commitPointer; + private long expirationDate; + private State state; + private Set changes; + /** Whether or not the COMMITTED change should be fully committed. */ + private boolean canCommit; + private TransactionType type; + private Set truncateInvalidTx; + private long truncateInvalidTxTime; + private long parentWritePointer; + private long[] checkpointPointers; + + // for Writable + public TransactionEdit() { + this.changes = Sets.newHashSet(); + this.truncateInvalidTx = Sets.newHashSet(); + } + + // package private for testing + TransactionEdit(long writePointer, long visibilityUpperBound, State state, long expirationDate, + Set changes, long commitPointer, boolean canCommit, TransactionType type, + Set truncateInvalidTx, long truncateInvalidTxTime, long parentWritePointer, + long[] checkpointPointers) { + this.writePointer = writePointer; + this.visibilityUpperBound = visibilityUpperBound; + this.state = state; + this.expirationDate = expirationDate; + this.changes = changes != null ? changes : Collections.emptySet(); + this.commitPointer = commitPointer; + this.canCommit = canCommit; + this.type = type; + this.truncateInvalidTx = truncateInvalidTx != null ? truncateInvalidTx : Collections.emptySet(); + this.truncateInvalidTxTime = truncateInvalidTxTime; + this.parentWritePointer = parentWritePointer; + this.checkpointPointers = checkpointPointers; + } + + /** + * Returns the transaction write pointer assigned for the state change. + */ + public long getWritePointer() { + return writePointer; + } + + void setWritePointer(long writePointer) { + this.writePointer = writePointer; + } + + public long getVisibilityUpperBound() { + return visibilityUpperBound; + } + + void setVisibilityUpperBound(long visibilityUpperBound) { + this.visibilityUpperBound = visibilityUpperBound; + } + + /** + * Returns the type of state change represented. + */ + public State getState() { + return state; + } + + void setState(State state) { + this.state = state; + } + + /** + * Returns any expiration timestamp (in milliseconds) associated with the state change. This should only + * be populated for changes of type {@link State#INPROGRESS}. + */ + public long getExpiration() { + return expirationDate; + } + + void setExpiration(long expirationDate) { + this.expirationDate = expirationDate; + } + + /** + * @return the set of changed row keys associated with the state change. This is only populated for edits + * of type {@link State#COMMITTING} or {@link State#COMMITTED}. + */ + public Set getChanges() { + return changes; + } + + void setChanges(Set changes) { + this.changes = changes; + } + + /** + * Returns the write pointer used to commit the row key change set. This is only populated for edits of type + * {@link State#COMMITTED}. + */ + public long getCommitPointer() { + return commitPointer; + } + + void setCommitPointer(long commitPointer) { + this.commitPointer = commitPointer; + } + + /** + * Returns whether or not the transaction should be moved to the committed set. This is only populated for edits + * of type {@link State#COMMITTED}. + */ + public boolean getCanCommit() { + return canCommit; + } + + void setCanCommit(boolean canCommit) { + this.canCommit = canCommit; + } + + /** + * Returns the transaction type. This is only populated for edits of type {@link State#INPROGRESS} or + * {@link State#ABORTED}. + */ + public TransactionType getType() { + return type; + } + + void setType(TransactionType type) { + this.type = type; + } + + /** + * Returns the transaction ids to be removed from invalid transaction list. This is only populated for + * edits of type {@link State#TRUNCATE_INVALID_TX} + */ + public Set getTruncateInvalidTx() { + return truncateInvalidTx; + } + + void setTruncateInvalidTx(Set truncateInvalidTx) { + this.truncateInvalidTx = truncateInvalidTx; + } + + /** + * Returns the time until which the invalid transactions need to be truncated from invalid transaction list. + * This is only populated for edits of type {@link State#TRUNCATE_INVALID_TX} + */ + public long getTruncateInvalidTxTime() { + return truncateInvalidTxTime; + } + + void setTruncateInvalidTxTime(long truncateInvalidTxTime) { + this.truncateInvalidTxTime = truncateInvalidTxTime; + } + + /** + * Returns the parent write pointer for a checkpoint operation. This is only populated for edits of type + * {@link State#CHECKPOINT} + */ + public long getParentWritePointer() { + return parentWritePointer; + } + + void setParentWritePointer(long parentWritePointer) { + this.parentWritePointer = parentWritePointer; + } + + /** + * Returns the checkpoint write pointers for the edit. This is only populated for edits of type + * {@link State#ABORTED}. + */ + public long[] getCheckpointPointers() { + return checkpointPointers; + } + + void setCheckpointPointers(long[] checkpointPointers) { + this.checkpointPointers = checkpointPointers; + } + + /** + * Creates a new instance in the {@link State#INPROGRESS} state. + */ + public static TransactionEdit createStarted(long writePointer, long visibilityUpperBound, + long expirationDate, TransactionType type) { + return new TransactionEdit(writePointer, visibilityUpperBound, State.INPROGRESS, + expirationDate, null, 0L, false, type, null, 0L, 0L, null); + } + + /** + * Creates a new instance in the {@link State#COMMITTING} state. + */ + public static TransactionEdit createCommitting(long writePointer, Set changes) { + return new TransactionEdit(writePointer, 0L, State.COMMITTING, 0L, changes, 0L, false, null, null, 0L, 0L, null); + } + + /** + * Creates a new instance in the {@link State#COMMITTED} state. + */ + public static TransactionEdit createCommitted(long writePointer, Set changes, long nextWritePointer, + boolean canCommit) { + return new TransactionEdit(writePointer, 0L, State.COMMITTED, 0L, changes, nextWritePointer, canCommit, null, + null, 0L, 0L, null); + } + + /** + * Creates a new instance in the {@link State#ABORTED} state. + */ + public static TransactionEdit createAborted(long writePointer, TransactionType type, long[] checkpointPointers) { + return new TransactionEdit(writePointer, 0L, State.ABORTED, 0L, null, 0L, false, type, null, 0L, 0L, + checkpointPointers); + } + + /** + * Creates a new instance in the {@link State#INVALID} state. + */ + public static TransactionEdit createInvalid(long writePointer) { + return new TransactionEdit(writePointer, 0L, State.INVALID, 0L, null, 0L, false, null, null, 0L, 0L, null); + } + + /** + * Creates a new instance in the {@link State#MOVE_WATERMARK} state. + */ + public static TransactionEdit createMoveWatermark(long writePointer) { + return new TransactionEdit(writePointer, 0L, State.MOVE_WATERMARK, 0L, null, 0L, false, null, null, 0L, 0L, null); + } + + /** + * Creates a new instance in the {@link State#TRUNCATE_INVALID_TX} state. + */ + public static TransactionEdit createTruncateInvalidTx(Set truncateInvalidTx) { + return new TransactionEdit(0L, 0L, State.TRUNCATE_INVALID_TX, 0L, null, 0L, false, null, truncateInvalidTx, + 0L, 0L, null); + } + + /** + * Creates a new instance in the {@link State#TRUNCATE_INVALID_TX} state. + */ + public static TransactionEdit createTruncateInvalidTxBefore(long truncateInvalidTxTime) { + return new TransactionEdit(0L, 0L, State.TRUNCATE_INVALID_TX, 0L, null, 0L, false, null, null, + truncateInvalidTxTime, 0L, null); + } + + /** + * Creates a new instance in the {@link State#CHECKPOINT} state. + */ + public static TransactionEdit createCheckpoint(long writePointer, long parentWritePointer) { + return new TransactionEdit(writePointer, 0L, State.CHECKPOINT, 0L, null, 0L, false, null, null, 0L, + parentWritePointer, null); + } + + @Override + public void write(DataOutput out) throws IOException { + TransactionEditCodecs.encode(this, out); + } + + @Override + public void readFields(DataInput in) throws IOException { + TransactionEditCodecs.decode(this, in); + } + + @Override + public final boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof TransactionEdit)) { + return false; + } + + TransactionEdit that = (TransactionEdit) o; + + return Objects.equal(this.writePointer, that.writePointer) && + Objects.equal(this.visibilityUpperBound, that.visibilityUpperBound) && + Objects.equal(this.commitPointer, that.commitPointer) && + Objects.equal(this.expirationDate, that.expirationDate) && + Objects.equal(this.state, that.state) && + Objects.equal(this.changes, that.changes) && + Objects.equal(this.canCommit, that.canCommit) && + Objects.equal(this.type, that.type) && + Objects.equal(this.truncateInvalidTx, that.truncateInvalidTx) && + Objects.equal(this.truncateInvalidTxTime, that.truncateInvalidTxTime) && + Objects.equal(this.parentWritePointer, that.parentWritePointer) && + Arrays.equals(this.checkpointPointers, that.checkpointPointers); + } + + @Override + public final int hashCode() { + return Objects.hashCode(writePointer, visibilityUpperBound, commitPointer, expirationDate, state, changes, + canCommit, type, parentWritePointer, checkpointPointers); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("writePointer", writePointer) + .add("visibilityUpperBound", visibilityUpperBound) + .add("commitPointer", commitPointer) + .add("expiration", expirationDate) + .add("state", state) + .add("changesSize", changes != null ? changes.size() : 0) + .add("canCommit", canCommit) + .add("type", type) + .add("truncateInvalidTx", truncateInvalidTx) + .add("truncateInvalidTxTime", truncateInvalidTxTime) + .add("parentWritePointer", parentWritePointer) + .add("checkpointPointers", checkpointPointers) + .toString(); + } + +} + diff --git a/cdap-data-fabric/src/main/java/io/cdap/cdap/data/dataset/SystemDatasetInstantiator.java b/cdap-data-fabric/src/main/java/io/cdap/cdap/data/dataset/SystemDatasetInstantiator.java index 406e01cd009d..02dd4c80d98c 100644 --- a/cdap-data-fabric/src/main/java/io/cdap/cdap/data/dataset/SystemDatasetInstantiator.java +++ b/cdap-data-fabric/src/main/java/io/cdap/cdap/data/dataset/SystemDatasetInstantiator.java @@ -16,7 +16,7 @@ package io.cdap.cdap.data.dataset; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.base.Throwables; import io.cdap.cdap.api.data.DatasetInstantiationException; import io.cdap.cdap.api.dataset.Dataset; @@ -74,7 +74,7 @@ public SystemDatasetInstantiator(DatasetFramework datasetFramework, this.classLoaderProvider = classLoaderProvider; this.datasetFramework = datasetFramework; this.parentClassLoader = parentClassLoader == null - ? Objects.firstNonNull(Thread.currentThread().getContextClassLoader(), + ? MoreObjects.firstNonNull(Thread.currentThread().getContextClassLoader(), getClass().getClassLoader()) : parentClassLoader; } diff --git a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/audit/DefaultAuditPublisher.java b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/audit/DefaultAuditPublisher.java index aa1d4c039388..035c9cbfe7d8 100644 --- a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/audit/DefaultAuditPublisher.java +++ b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/audit/DefaultAuditPublisher.java @@ -16,7 +16,7 @@ package io.cdap.cdap.data2.audit; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.gson.Gson; import com.google.inject.Inject; import io.cdap.cdap.api.messaging.TopicNotFoundException; @@ -69,7 +69,7 @@ public void publish(EntityId entityId, AuditType auditType, AuditPayload auditPa @Override public void publish(MetadataEntity metadataEntity, AuditType auditType, AuditPayload auditPayload) { - String userId = Objects.firstNonNull(SecurityRequestContext.getUserId(), ""); + String userId = MoreObjects.firstNonNull(SecurityRequestContext.getUserId(), ""); AuditMessage auditMessage = new AuditMessage(System.currentTimeMillis(), metadataEntity, userId, auditType, auditPayload); LOG.trace("Publishing audit message {}", auditMessage); diff --git a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/datafabric/dataset/RemoteDatasetFramework.java b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/datafabric/dataset/RemoteDatasetFramework.java index 14e0fbca6c3d..8013c508891f 100644 --- a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/datafabric/dataset/RemoteDatasetFramework.java +++ b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/datafabric/dataset/RemoteDatasetFramework.java @@ -16,7 +16,7 @@ package io.cdap.cdap.data2.datafabric.dataset; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.base.Throwables; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; @@ -380,7 +380,7 @@ private T getType(DatasetTypeMeta datasetTypeMeta, DatasetClassLoaderProvider classLoaderProvider) { if (classLoader == null) { - classLoader = Objects.firstNonNull(Thread.currentThread().getContextClassLoader(), + classLoader = MoreObjects.firstNonNull(Thread.currentThread().getContextClassLoader(), getClass().getClassLoader()); } diff --git a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/datafabric/dataset/service/AuthorizationDatasetTypeService.java b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/datafabric/dataset/service/AuthorizationDatasetTypeService.java index c3510fc0022b..31f4ca50d80a 100644 --- a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/datafabric/dataset/service/AuthorizationDatasetTypeService.java +++ b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/datafabric/dataset/service/AuthorizationDatasetTypeService.java @@ -56,12 +56,12 @@ public AuthorizationDatasetTypeService( @Override protected void startUp() throws Exception { - delegate.startAndWait(); + delegate.startAsync().awaitRunning(); } @Override protected void shutDown() throws Exception { - delegate.stopAndWait(); + delegate.stopAsync().awaitTerminated(); } @Override diff --git a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/datafabric/dataset/service/DatasetService.java b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/datafabric/dataset/service/DatasetService.java index d6e2971dd2d9..c2914624d19a 100644 --- a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/datafabric/dataset/service/DatasetService.java +++ b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/datafabric/dataset/service/DatasetService.java @@ -16,7 +16,7 @@ package io.cdap.cdap.data2.datafabric.dataset.service; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.collect.Iterables; import com.google.common.util.concurrent.AbstractService; import com.google.common.util.concurrent.MoreExecutors; @@ -142,7 +142,7 @@ protected void doStop() { private void startUp() { try { LOG.info("Starting DatasetService..."); - typeService.startAndWait(); + typeService.startAsync().awaitRunning(); httpService.start(); // setting watch for ops executor service that we need to be running to operate correctly @@ -155,10 +155,10 @@ private void startUp() { LOG.info("Discovered {} service", Constants.Service.DATASET_EXECUTOR); opExecutorDiscovered.set(serviceDiscovered); } - }, MoreExecutors.sameThreadExecutor()); + }, MoreExecutors.directExecutor()); for (DatasetMetricsReporter metricsReporter : metricReporters) { - metricsReporter.start(); + metricsReporter.startAsync(); } } catch (Throwable t) { notifyFailed(t); @@ -234,14 +234,14 @@ private void doShutdown() throws Exception { } for (DatasetMetricsReporter metricsReporter : metricReporters) { - metricsReporter.stop(); + metricsReporter.stopAsync(); } if (opExecutorServiceWatch != null) { opExecutorServiceWatch.cancel(); } - typeService.stopAndWait(); + typeService.stopAsync().awaitTerminated(); // Wait for a few seconds for requests to stop httpService.stop(); @@ -250,7 +250,7 @@ private void doShutdown() throws Exception { @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("bindAddress", httpService.getBindAddress()) .toString(); } diff --git a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/datafabric/dataset/service/DefaultDatasetTypeService.java b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/datafabric/dataset/service/DefaultDatasetTypeService.java index 03be04d8481b..93438c0bad5a 100644 --- a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/datafabric/dataset/service/DefaultDatasetTypeService.java +++ b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/datafabric/dataset/service/DefaultDatasetTypeService.java @@ -108,7 +108,7 @@ public DefaultDatasetTypeService(DatasetTypeManager typeManager, @Override protected void startUp() throws Exception { - txClientService.startAndWait(); + txClientService.startAsync().awaitRunning(); deleteSystemModules(); deployDefaultModules(); if (!extensionModules.isEmpty()) { @@ -118,7 +118,7 @@ protected void startUp() throws Exception { @Override protected void shutDown() throws Exception { - txClientService.stopAndWait(); + txClientService.stopAsync().awaitTerminated(); } /** @@ -322,7 +322,9 @@ protected void onFinish(HttpResponder responder, File uploadedFile) throws Excep Locations.mkdirsIfNotExists(archiveDir); LOG.debug("Copy from {} to {}", uploadedFile, tmpLocation); - Files.copy(uploadedFile, Locations.newOutputSupplier(tmpLocation)); + try (java.io.OutputStream os = tmpLocation.getOutputStream()) { + Files.copy(uploadedFile, os); + } // Finally, move archive to final location LOG.debug("Storing module {} jar at {}", datasetModuleId, archive); diff --git a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/datafabric/dataset/service/executor/DatasetAdminService.java b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/datafabric/dataset/service/executor/DatasetAdminService.java index 513e26cf05cb..e6747c102931 100644 --- a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/datafabric/dataset/service/executor/DatasetAdminService.java +++ b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/datafabric/dataset/service/executor/DatasetAdminService.java @@ -138,7 +138,13 @@ public DatasetCreationResponse createOrUpdate(final DatasetId datasetInstanceId, typeMeta.getName()); } } finally { - Closeables.closeQuietly(admin); + try { + if (admin != null) { + admin.close(); + } + } catch (Exception e) { + // Ignore + } } return spec1; }); @@ -223,7 +229,13 @@ public void drop(final DatasetId datasetInstanceId, final DatasetTypeMeta typeMe try { admin.drop(); } finally { - Closeables.closeQuietly(admin); + try { + if (admin != null) { + admin.close(); + } + } catch (Exception e) { + // Ignore + } } return null; }); @@ -267,7 +279,13 @@ private T performDatasetAdmin(final DatasetId datasetInstanceId, Operation notification) { ClassLoader cl = notification.getValue(); if (cl instanceof Closeable) { - Closeables.closeQuietly((Closeable) cl); + try { + ((Closeable) cl).close(); + } catch (IOException e) { + // Ignore + } } } } @@ -122,12 +125,12 @@ public boolean equals(Object o) { CacheKey that = (CacheKey) o; - return Objects.equal(this.uri, that.uri); + return Objects.equals(this.uri, that.uri); } @Override public int hashCode() { - return Objects.hashCode(uri); + return Objects.hash(uri); } } diff --git a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/dataset2/DatasetDefinitionRegistries.java b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/dataset2/DatasetDefinitionRegistries.java index 9022bffad52c..135da2bf3906 100644 --- a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/dataset2/DatasetDefinitionRegistries.java +++ b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/dataset2/DatasetDefinitionRegistries.java @@ -16,7 +16,7 @@ package io.cdap.cdap.data2.dataset2; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import io.cdap.cdap.api.dataset.module.DatasetDefinitionRegistry; import io.cdap.cdap.api.dataset.module.DatasetModule; import io.cdap.cdap.common.lang.ClassLoaders; @@ -35,7 +35,7 @@ public static void register(String moduleClassName, ClassLoader systemClassLoader = DatasetDefinitionRegistries.class.getClassLoader(); // Either uses the given classloader or the system one - ClassLoader moduleClassLoader = Objects.firstNonNull(classLoader, systemClassLoader); + ClassLoader moduleClassLoader = MoreObjects.firstNonNull(classLoader, systemClassLoader); Class moduleClass; try { diff --git a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/dataset2/DynamicDatasetCache.java b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/dataset2/DynamicDatasetCache.java index 5af816043870..939e77bcf552 100644 --- a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/dataset2/DynamicDatasetCache.java +++ b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/dataset2/DynamicDatasetCache.java @@ -16,9 +16,9 @@ package io.cdap.cdap.data2.dataset2; -import com.google.common.base.Objects; -import com.google.common.io.Closeables; +import com.google.common.base.MoreObjects; import io.cdap.cdap.api.common.RuntimeArguments; +import java.util.Objects; import io.cdap.cdap.api.common.Scope; import io.cdap.cdap.api.data.DatasetContext; import io.cdap.cdap.api.data.DatasetInstantiationException; @@ -300,7 +300,13 @@ protected abstract T getDataset(DatasetCacheKey key, boolean */ @Override public void close() { - Closeables.closeQuietly(instantiator); + try { + if (instantiator != null) { + instantiator.close(); + } + } catch (Exception e) { + // Ignore + } } /** @@ -366,19 +372,19 @@ public boolean equals(Object o) { // Omit accessType here since we don't have to request a another dataset instance just because // of the different accessType. Same for the hashCode() method - return Objects.equal(this.namespace, that.namespace) - && Objects.equal(this.name, that.name) - && Objects.equal(this.arguments, that.arguments); + return Objects.equals(this.namespace, that.namespace) + && Objects.equals(this.name, that.name) + && Objects.equals(this.arguments, that.arguments); } @Override public int hashCode() { - return Objects.hashCode(namespace, name, arguments); + return Objects.hash(namespace, name, arguments); } @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("namespace", namespace) .add("name", name) .add("arguments", arguments) diff --git a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/dataset2/SingleThreadDatasetCache.java b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/dataset2/SingleThreadDatasetCache.java index b9b2a6237904..b4d6b9e27817 100644 --- a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/dataset2/SingleThreadDatasetCache.java +++ b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/dataset2/SingleThreadDatasetCache.java @@ -24,9 +24,7 @@ import com.google.common.cache.RemovalNotification; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; -import com.google.common.io.Closeables; import com.google.common.util.concurrent.UncheckedExecutionException; -import io.cdap.cdap.api.data.DatasetContext; import io.cdap.cdap.api.data.DatasetInstantiationException; import io.cdap.cdap.api.dataset.Dataset; import io.cdap.cdap.api.dataset.metrics.MeteredDataset; @@ -39,6 +37,7 @@ import io.cdap.cdap.proto.id.DatasetId; import io.cdap.cdap.proto.id.NamespaceId; import java.io.Closeable; +import java.io.IOException; import java.util.Collections; import java.util.Deque; import java.util.EnumSet; @@ -367,7 +366,11 @@ public void invalidate() { public void close() { for (TransactionAware txAware : extraTxAwares) { if (txAware instanceof Closeable) { - Closeables.closeQuietly((Closeable) txAware); + try { + ((Closeable) txAware).close(); + } catch (IOException e) { + // Ignore + } } } invalidate(); diff --git a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/dataset2/lib/table/leveldb/LevelDBTableService.java b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/dataset2/lib/table/leveldb/LevelDBTableService.java index f1cae276603d..daece86cbca9 100644 --- a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/dataset2/lib/table/leveldb/LevelDBTableService.java +++ b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/dataset2/lib/table/leveldb/LevelDBTableService.java @@ -24,7 +24,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; -import com.google.common.io.Closeables; + import com.google.inject.Inject; import com.google.inject.Singleton; import io.cdap.cdap.common.conf.CConfiguration; @@ -212,7 +212,13 @@ public void compact(String tableName) { */ public void clearTables() { for (DB entries : tables.values()) { - Closeables.closeQuietly(entries); + try { + if (entries != null) { + entries.close(); + } + } catch (Exception e) { + // Ignore + } } tables.clear(); } diff --git a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/dataset2/lib/timeseries/EntityTable.java b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/dataset2/lib/timeseries/EntityTable.java index 13dd05ff715d..2ae473ab2b34 100644 --- a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/dataset2/lib/timeseries/EntityTable.java +++ b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/dataset2/lib/timeseries/EntityTable.java @@ -15,7 +15,7 @@ */ package io.cdap.cdap.data2.dataset2.lib.timeseries; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; @@ -293,7 +293,7 @@ public int hashCode() { @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("type", type) .add("name", name) .toString(); diff --git a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/queue/ConsumerConfig.java b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/queue/ConsumerConfig.java index 468093f9c826..f27e35783abf 100644 --- a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/queue/ConsumerConfig.java +++ b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/queue/ConsumerConfig.java @@ -15,8 +15,9 @@ */ package io.cdap.cdap.data2.queue; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; +import java.util.Objects; /** * Contains queue consumer instance configuration. @@ -52,7 +53,7 @@ public int getInstanceId() { @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("groupId", getGroupId()) .add("instanceId", instanceId) .add("groupSize", getGroupSize()) @@ -76,6 +77,6 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hashCode(super.hashCode(), instanceId); + return Objects.hash(super.hashCode(), instanceId); } } diff --git a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/queue/ConsumerGroupConfig.java b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/queue/ConsumerGroupConfig.java index c169d0871b45..7ec72a844c46 100644 --- a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/queue/ConsumerGroupConfig.java +++ b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/queue/ConsumerGroupConfig.java @@ -16,7 +16,8 @@ package io.cdap.cdap.data2.queue; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; +import java.util.Objects; /** * Contains queue consumer group information. @@ -58,7 +59,7 @@ public String getHashKey() { @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("groupId", groupId) .add("groupSize", groupSize) .add("dequeueStrategy", dequeueStrategy) @@ -80,11 +81,11 @@ public boolean equals(Object o) { return groupId == other.groupId && groupSize == other.groupSize && dequeueStrategy == other.dequeueStrategy - && Objects.equal(hashKey, other.hashKey); + && Objects.equals(hashKey, other.hashKey); } @Override public int hashCode() { - return Objects.hashCode(groupId, groupSize, dequeueStrategy, hashKey); + return Objects.hash(groupId, groupSize, dequeueStrategy, hashKey); } } diff --git a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/queue/DequeueResult.java b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/queue/DequeueResult.java index 38704b937a19..f6ebca212571 100644 --- a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/queue/DequeueResult.java +++ b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/queue/DequeueResult.java @@ -16,7 +16,7 @@ package io.cdap.cdap.data2.queue; -import com.google.common.collect.Iterators; + import java.util.Iterator; @@ -88,7 +88,7 @@ public int size() { @Override public Iterator iterator() { - return Iterators.emptyIterator(); + return java.util.Collections.emptyIterator(); } }; } diff --git a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/transaction/DynamicTransactionExecutor.java b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/transaction/DynamicTransactionExecutor.java index 33f3e845a415..00dbc78fa8e1 100644 --- a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/transaction/DynamicTransactionExecutor.java +++ b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/transaction/DynamicTransactionExecutor.java @@ -48,7 +48,7 @@ public class DynamicTransactionExecutor extends AbstractTransactionExecutor { public DynamicTransactionExecutor(TransactionContextFactory txContextFactory, RetryStrategy retryStrategy) { - super(MoreExecutors.sameThreadExecutor()); + super(MoreExecutors.newDirectExecutorService()); this.txContextFactory = txContextFactory; this.retryStrategy = retryStrategy; } diff --git a/cdap-data-fabric/src/main/java/io/cdap/cdap/spi/metadata/dataset/SearchHelper.java b/cdap-data-fabric/src/main/java/io/cdap/cdap/spi/metadata/dataset/SearchHelper.java index 1d81a2c91614..0819cab1a183 100644 --- a/cdap-data-fabric/src/main/java/io/cdap/cdap/spi/metadata/dataset/SearchHelper.java +++ b/cdap-data-fabric/src/main/java/io/cdap/cdap/spi/metadata/dataset/SearchHelper.java @@ -22,7 +22,7 @@ import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.io.Closeables; + import com.google.inject.Inject; import com.google.inject.name.Named; import io.cdap.cdap.api.Transactional; @@ -198,7 +198,14 @@ public void discardDataset(Dataset dataset) { @Override public void close() { for (String scope : datasets.keySet()) { - Closeables.closeQuietly(datasets.get(scope)); + try { + MetadataDataset dataset = datasets.get(scope); + if (dataset != null) { + dataset.close(); + } + } catch (Exception e) { + // Ignore + } } datasets.clear(); } diff --git a/cdap-data-fabric/src/main/java/org/apache/tephra/DefaultTransactionExecutor.java b/cdap-data-fabric/src/main/java/org/apache/tephra/DefaultTransactionExecutor.java new file mode 100644 index 000000000000..d3c182dd5b16 --- /dev/null +++ b/cdap-data-fabric/src/main/java/org/apache/tephra/DefaultTransactionExecutor.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tephra; + +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.inject.Inject; +import com.google.inject.assistedinject.Assisted; + +import java.util.Arrays; +import java.util.Collection; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; + +/** + * Utility class that encapsulates the transaction life cycle over a given set of + * transaction-aware datasets. The executor can be reused across multiple invocations + * of the execute() method. However, it is not thread-safe for concurrent execution. + *

+ * Transaction execution will be retries according to specified in constructor {@link RetryStrategy}. + * By default {@link RetryOnConflictStrategy} is used with max 20 retries and 100 ms between retries. + *

+ */ +public class DefaultTransactionExecutor extends AbstractTransactionExecutor { + + private final Collection txAwares; + private final TransactionSystemClient txClient; + private final RetryStrategy retryStrategy; + + /** + * Convenience constructor, has same affect as {@link #DefaultTransactionExecutor(TransactionSystemClient, Iterable)} + */ + public DefaultTransactionExecutor(TransactionSystemClient txClient, TransactionAware... txAwares) { + this(txClient, Arrays.asList(txAwares)); + } + + + public DefaultTransactionExecutor(TransactionSystemClient txClient, + Iterable txAwares, + RetryStrategy retryStrategy) { + + super(MoreExecutors.newDirectExecutorService()); + this.txAwares = ImmutableList.copyOf(txAwares); + this.txClient = txClient; + this.retryStrategy = retryStrategy; + } + + /** + * Constructor for a transaction executor. + */ + @Inject + public DefaultTransactionExecutor(TransactionSystemClient txClient, @Assisted Iterable txAwares) { + this(txClient, txAwares, RetryStrategies.retryOnConflict(20, 100)); + } + + @Override + public O execute(Function function, I input) throws TransactionFailureException, InterruptedException { + return executeWithRetry(function, input); + } + + @Override + public void execute(final Procedure procedure, I input) + throws TransactionFailureException, InterruptedException { + + execute(new Function() { + @Override + public Void apply(I input) throws Exception { + procedure.apply(input); + return null; + } + }, input); + } + + @Override + public O execute(final Callable callable) throws TransactionFailureException, InterruptedException { + return execute(new Function() { + @Override + public O apply(Void input) throws Exception { + return callable.call(); + } + }, null); + } + + @Override + public void execute(final Subroutine subroutine) throws TransactionFailureException, InterruptedException { + execute(new Function() { + @Override + public Void apply(Void input) throws Exception { + subroutine.apply(); + return null; + } + }, null); + } + + private O executeWithRetry(Function function, I input) + throws TransactionFailureException, InterruptedException { + + int retries = 0; + while (true) { + try { + return executeOnce(function, input); + } catch (TransactionFailureException e) { + long delay = retryStrategy.nextRetry(e, ++retries); + + if (delay < 0) { + throw e; + } + + if (delay > 0) { + TimeUnit.MILLISECONDS.sleep(delay); + } + } + } + + } + + private O executeOnce(Function function, I input) throws TransactionFailureException { + TransactionContext txContext = new TransactionContext(txClient, txAwares); + txContext.start(); + O o = null; + try { + o = function.apply(input); + } catch (Throwable e) { + txContext.abort(new TransactionFailureException("Transaction function failure for transaction. ", e)); + // abort will throw + } + // will throw if smth goes wrong + txContext.finish(); + return o; + } +} diff --git a/cdap-data-fabric/src/main/java/org/apache/tephra/TransactionManager.java b/cdap-data-fabric/src/main/java/org/apache/tephra/TransactionManager.java new file mode 100644 index 000000000000..34ef75068ee0 --- /dev/null +++ b/cdap-data-fabric/src/main/java/org/apache/tephra/TransactionManager.java @@ -0,0 +1,1622 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tephra; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.MoreObjects; +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import com.google.common.base.Stopwatch; +import com.google.common.base.Throwables; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.AbstractService; +import com.google.inject.Inject; +import it.unimi.dsi.fastutil.longs.LongArrayList; +import it.unimi.dsi.fastutil.longs.LongArraySet; +import it.unimi.dsi.fastutil.longs.LongIterator; +import it.unimi.dsi.fastutil.longs.LongSet; +import org.apache.hadoop.conf.Configuration; +import org.apache.tephra.manager.InvalidTxList; +import org.apache.tephra.metrics.DefaultMetricsCollector; +import org.apache.tephra.metrics.MetricsCollector; +import org.apache.tephra.persist.NoOpTransactionStateStorage; +import org.apache.tephra.persist.TransactionEdit; +import org.apache.tephra.persist.TransactionLog; +import org.apache.tephra.persist.TransactionLogReader; +import org.apache.tephra.persist.TransactionSnapshot; +import org.apache.tephra.persist.TransactionStateStorage; +import org.apache.tephra.snapshot.SnapshotCodecProvider; +import org.apache.tephra.util.TxUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Set; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * This is the central place to manage all active transactions in the system. + * + * A transaction consists of + *
    + *
  • A write pointer: This is the version used for all writes of that transaction.
    • + *
  • A read pointer: All reads under the transaction use this as an upper bound for the version.
    • + *
  • A set of excluded versions: These are the write versions of other transactions that must be excluded from + * reads, because those transactions are still in progress, or they failed but couldn't be properly rolled back.
    • + *
+ * To use the transaction system, a client must follow this sequence of steps: + *
    + *
  1. Request a new transaction.
    2. + *
  2. Use the transaction to read and write datasets. Datasets are encouraged to cache the writes of the + * transaction in memory, to reduce the cost of rollback in case the transaction fails.
    3. + *
  3. Check whether the transaction has conflicts. For this, the set of change keys are submitted via canCommit(), + * and the transaction manager verifies that none of these keys are in conflict with other transactions that + * committed since the start of this transaction.
    4. + *
  4. If the transaction has conflicts: + *
      + *
    1. Roll back the changes in every dataset that was changed. This can happen in-memory if the + * changes were cached.
      2. + *
    2. Abort the transaction to remove it from the active transactions.
      3. + *
    + *
  5. If the transaction has no conflicts:
    6. + *
      + *
    1. Persist all datasets changes to storage.
      2. + *
    2. Commit the transaction. This will repeat the conflict detection, because more overlapping transactions + * may have committed since the first conflict check.
      3. + *
    3. If the transaction has conflicts:
      4. + *
        + *
      1. Roll back the changes in every dataset that was changed. This is more expensive because + * changes must be undone in persistent storage.
        2. + *
      2. Abort the transaction to remove it from the active transactions.
        3. + *
      + *
    + *
+ * Transactions may be short or long-running. A short transaction is started with a timeout, and if it is not + * committed before that timeout, it is invalidated and excluded from future reads. A long-running transaction has + * no timeout and will remain active until it is committed or aborted. Long transactions are typically used in + * map/reduce jobs and can produce enormous amounts of changes. Therefore, long transactions do not participate in + * conflict detection (they would almost always have conflicts). We also assume that the changes of long transactions + * are not tracked, and therefore cannot be rolled back. Hence, when a long transaction is aborted, it remains in the + * list of excluded transactions to make its writes invisible. + */ +public class TransactionManager extends AbstractService { + // todo: optimize heavily + + private static final Logger LOG = LoggerFactory.getLogger(TransactionManager.class); + + // poll every 1 second to check whether a snapshot is needed + private static final long SNAPSHOT_POLL_INTERVAL = 1000L; + + //poll every 10 second to emit metrics + private static final long METRICS_POLL_INTERVAL = 10000L; + + //Client id that is used if a client doesn't provide one while starting a transaction. + private static final String DEFAULT_CLIENTID = "unknown"; + + // Transactions that are in progress, with their info. + private final NavigableMap inProgress = new ConcurrentSkipListMap<>(); + + // the list of transactions that are invalid (not properly committed/aborted, or timed out) + private final InvalidTxList invalidTxList = new InvalidTxList(); + + // todo: use moving array instead (use Long2ObjectMap in fastutil) + // todo: should this be consolidated with inProgress? + // commit time next writePointer -> changes made by this tx + private final NavigableMap committedChangeSets = new ConcurrentSkipListMap<>(); + // not committed yet + private final Map committingChangeSets = Maps.newConcurrentMap(); + + private long readPointer; + private long lastWritePointer; + private MetricsCollector txMetricsCollector; + + private final TransactionStateStorage persistor; + + private final int cleanupInterval; + private final int defaultTimeout; + private final int defaultLongTimeout; + private final int maxTimeout; + private DaemonThreadExecutor cleanupThread = null; + + private volatile TransactionLog currentLog; + + // timestamp of the last completed snapshot + private long lastSnapshotTime; + // frequency in millis to perform snapshots + private final long snapshotFrequencyInSeconds; + // number of most recent snapshots to retain + private final int snapshotRetainCount; + private DaemonThreadExecutor snapshotThread; + private DaemonThreadExecutor metricsThread; + + // retention of client id for transactions - this affects memory footprint + private final boolean retainClientId; + private final boolean retainClientIdPastCommit; + + // lock guarding change of the current transaction log + private final ReentrantReadWriteLock logLock = new ReentrantReadWriteLock(); + private final Lock logReadLock = logLock.readLock(); + private final Lock logWriteLock = logLock.writeLock(); + + private final int changeSetCountLimit; + private final int changeSetCountThreshold; + private final long changeSetSizeLimit; + private final long changeSetSizeThreshold; + + // fudge factor (in milliseconds) used when interpreting transactions as LONG based on expiration + // TODO: REMOVE WITH txnBackwardsCompatCheck() + private final long longTimeoutTolerance; + + public TransactionManager(Configuration config) { + this(config, new NoOpTransactionStateStorage(new SnapshotCodecProvider(config)), new DefaultMetricsCollector()); + } + + @Inject + public TransactionManager(Configuration conf, @Nonnull TransactionStateStorage persistor, + MetricsCollector txMetricsCollector) { + this.persistor = persistor; + cleanupInterval = conf.getInt(TxConstants.Manager.CFG_TX_CLEANUP_INTERVAL, + TxConstants.Manager.DEFAULT_TX_CLEANUP_INTERVAL); + maxTimeout = conf.getInt(TxConstants.Manager.CFG_TX_MAX_TIMEOUT, + TxConstants.Manager.DEFAULT_TX_MAX_TIMEOUT); + defaultTimeout = conf.getInt(TxConstants.Manager.CFG_TX_TIMEOUT, + TxConstants.Manager.DEFAULT_TX_TIMEOUT); + defaultLongTimeout = conf.getInt(TxConstants.Manager.CFG_TX_LONG_TIMEOUT, + TxConstants.Manager.DEFAULT_TX_LONG_TIMEOUT); + snapshotFrequencyInSeconds = conf.getLong(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL, + TxConstants.Manager.DEFAULT_TX_SNAPSHOT_INTERVAL); + // must always keep at least 1 snapshot + snapshotRetainCount = Math.max(conf.getInt(TxConstants.Manager.CFG_TX_SNAPSHOT_RETAIN, + TxConstants.Manager.DEFAULT_TX_SNAPSHOT_RETAIN), 1); + + changeSetCountLimit = conf.getInt(TxConstants.Manager.CFG_TX_CHANGESET_COUNT_LIMIT, + TxConstants.Manager.DEFAULT_TX_CHANGESET_COUNT_LIMIT); + changeSetCountThreshold = conf.getInt(TxConstants.Manager.CFG_TX_CHANGESET_COUNT_WARN_THRESHOLD, + TxConstants.Manager.DEFAULT_TX_CHANGESET_COUNT_WARN_THRESHOLD); + changeSetSizeLimit = conf.getLong(TxConstants.Manager.CFG_TX_CHANGESET_SIZE_LIMIT, + TxConstants.Manager.DEFAULT_TX_CHANGESET_SIZE_LIMIT); + changeSetSizeThreshold = conf.getLong(TxConstants.Manager.CFG_TX_CHANGESET_SIZE_WARN_THRESHOLD, + TxConstants.Manager.DEFAULT_TX_CHANGESET_SIZE_WARN_THRESHOLD); + + // intentionally not using a constant, as this config should not be exposed + // TODO: REMOVE WITH txnBackwardsCompatCheck() + longTimeoutTolerance = conf.getLong("data.tx.long.timeout.tolerance", 10000); + + ClientIdRetention retention = ClientIdRetention.valueOf( + conf.get(TxConstants.Manager.CFG_TX_RETAIN_CLIENT_ID, + TxConstants.Manager.DEFAULT_TX_RETAIN_CLIENT_ID).toUpperCase()); + this.retainClientId = retention != ClientIdRetention.OFF; + this.retainClientIdPastCommit = retention == ClientIdRetention.COMMITTED; + + this.txMetricsCollector = txMetricsCollector; + this.txMetricsCollector.configure(conf); + clear(); + } + + enum ClientIdRetention { OFF, ACTIVE, COMMITTED } + + private void clear() { + invalidTxList.clear(); + inProgress.clear(); + committedChangeSets.clear(); + committingChangeSets.clear(); + lastWritePointer = 0; + readPointer = 0; + lastSnapshotTime = 0; + } + + private boolean isStopping() { + return State.STOPPING.equals(state()); + } + + @Override + public synchronized void doStart() { + LOG.info("Starting transaction manager."); + txMetricsCollector.startAsync().awaitRunning(); + // start up the persistor + persistor.startAsync().awaitRunning(); + try { + persistor.setupStorage(); + } catch (IOException e) { + Throwables.propagate(e); + } + // establish defaults in case there is no persistence + clear(); + // attempt to recover state from last run + recoverState(); + // start the periodic cleanup thread + startCleanupThread(); + startSnapshotThread(); + startMetricsThread(); + // initialize the WAL if we did not force a snapshot in recoverState() + initLog(); + // initialize next write pointer if needed + if (lastWritePointer == 0) { + lastWritePointer = getNextWritePointer(); + readPointer = lastWritePointer; + } + + notifyStarted(); + } + + private void initLog() { + if (currentLog == null) { + try { + currentLog = persistor.createLog(System.currentTimeMillis()); + } catch (IOException ioe) { + throw Throwables.propagate(ioe); + } + } + } + + private void startCleanupThread() { + if (cleanupInterval <= 0 || defaultTimeout <= 0) { + return; + } + LOG.info("Starting periodic timed-out transaction cleanup every " + cleanupInterval + + " seconds with default timeout of " + defaultTimeout + " seconds."); + this.cleanupThread = new DaemonThreadExecutor("tx-clean-timeout") { + @Override + public void doRun() { + cleanupTimedOutTransactions(); + } + + @Override + public long getSleepMillis() { + return cleanupInterval * 1000; + } + }; + cleanupThread.start(); + } + + private void startSnapshotThread() { + if (snapshotFrequencyInSeconds > 0) { + LOG.info("Starting periodic snapshot thread, frequency = " + snapshotFrequencyInSeconds + + " seconds, location = " + persistor.getLocation()); + this.snapshotThread = new DaemonThreadExecutor("tx-snapshot") { + @Override + public void doRun() { + long currentTime = System.currentTimeMillis(); + if (lastSnapshotTime < (currentTime - snapshotFrequencyInSeconds * 1000)) { + try { + doSnapshot(false); + } catch (IOException ioe) { + LOG.error("Periodic snapshot failed!", ioe); + } + } + } + + @Override + protected void onShutdown() { + // perform a final snapshot + try { + LOG.info("Writing final snapshot prior to shutdown"); + doSnapshot(true); + } catch (IOException ioe) { + LOG.error("Failed performing final snapshot on shutdown", ioe); + } + } + + @Override + public long getSleepMillis() { + return SNAPSHOT_POLL_INTERVAL; + } + }; + snapshotThread.start(); + } + } + + // Emits Transaction Data structures size as metrics + private void startMetricsThread() { + LOG.info("Starting periodic Metrics Emitter thread, frequency = " + METRICS_POLL_INTERVAL); + this.metricsThread = new DaemonThreadExecutor("tx-metrics") { + @Override + public void doRun() { + txMetricsCollector.gauge("committing.size", committingChangeSets.size()); + txMetricsCollector.gauge("committed.size", committedChangeSets.size()); + txMetricsCollector.gauge("inprogress.size", inProgress.size()); + txMetricsCollector.gauge("invalid.size", getInvalidSize()); + } + + @Override + protected void onShutdown() { + // perform a final metrics emit + txMetricsCollector.gauge("committing.size", committingChangeSets.size()); + txMetricsCollector.gauge("committed.size", committedChangeSets.size()); + txMetricsCollector.gauge("inprogress.size", inProgress.size()); + txMetricsCollector.gauge("invalid.size", getInvalidSize()); + } + + @Override + public long getSleepMillis() { + return METRICS_POLL_INTERVAL; + } + }; + metricsThread.start(); + } + + private void cleanupTimedOutTransactions() { + List invalidEdits = null; + logReadLock.lock(); + try { + synchronized (this) { + if (!isRunning()) { + return; + } + long currentTime = System.currentTimeMillis(); + Map timedOut = Maps.newHashMap(); + for (Map.Entry tx : inProgress.entrySet()) { + InProgressTx inProgressTx = tx.getValue(); + long expiration = inProgressTx.getExpiration(); + if (expiration >= 0L && currentTime > expiration) { + // timed out, remember tx id (can't remove while iterating over entries) + timedOut.put(tx.getKey(), inProgressTx.getType()); + LOG.info("Tx invalid list: added tx {} belonging to client '{}' because of timeout.", + tx.getKey(), inProgressTx.getClientId()); + } else if (expiration < 0) { + LOG.warn("Transaction {} has negative expiration time {}. Likely cause is the transaction was not " + + "migrated correctly, this transaction will be expired immediately", tx.getKey(), expiration); + timedOut.put(tx.getKey(), InProgressType.LONG); + } + } + if (!timedOut.isEmpty()) { + invalidEdits = Lists.newArrayListWithCapacity(timedOut.size()); + invalidTxList.addAll(timedOut.keySet()); + for (Map.Entry tx : timedOut.entrySet()) { + inProgress.remove(tx.getKey()); + // checkpoints never go into the committing change sets or the edits + if (!InProgressType.CHECKPOINT.equals(tx.getValue())) { + committingChangeSets.remove(tx.getKey()); + invalidEdits.add(TransactionEdit.createInvalid(tx.getKey())); + } + } + + LOG.info("Invalidated {} transactions due to timeout.", timedOut.size()); + } + } + if (invalidEdits != null) { + appendToLog(invalidEdits); + } + } finally { + this.logReadLock.unlock(); + } + } + + public synchronized TransactionSnapshot getSnapshot() throws IOException { + TransactionSnapshot snapshot; + if (!isRunning() && !isStopping()) { + return null; + } + + long now = System.currentTimeMillis(); + // avoid duplicate snapshots at same timestamp + if (now == lastSnapshotTime || (currentLog != null && now == currentLog.getTimestamp())) { + try { + TimeUnit.MILLISECONDS.sleep(1); + } catch (InterruptedException ie) { } + } + // copy in memory state + snapshot = getCurrentState(); + + LOG.debug("Starting snapshot of transaction state with timestamp {}", snapshot.getTimestamp()); + LOG.debug("Returning snapshot of state: " + snapshot); + return snapshot; + } + + /** + * Take a snapshot of the transaction state and serialize it into the given output stream. + * @return whether a snapshot was taken. + */ + public boolean takeSnapshot(OutputStream out) throws IOException { + TransactionSnapshot snapshot = getSnapshot(); + if (snapshot != null) { + persistor.writeSnapshot(out, snapshot); + return true; + } else { + return false; + } + } + + private void doSnapshot(boolean closing) throws IOException { + long snapshotTime = 0L; + TransactionSnapshot snapshot; + TransactionLog oldLog; + try { + this.logWriteLock.lock(); + try { + synchronized (this) { + snapshot = getSnapshot(); + if (snapshot == null && !closing) { + return; + } + if (snapshot != null) { + snapshotTime = snapshot.getTimestamp(); + } + + // roll WAL + oldLog = currentLog; + if (!closing) { + currentLog = persistor.createLog(snapshot.getTimestamp()); + } + } + // there may not be an existing log on startup + if (oldLog != null) { + oldLog.close(); + } + } finally { + this.logWriteLock.unlock(); + } + + // save snapshot + if (snapshot != null) { + persistor.writeSnapshot(snapshot); + lastSnapshotTime = snapshotTime; + + // clean any obsoleted snapshots and WALs + long oldestRetainedTimestamp = persistor.deleteOldSnapshots(snapshotRetainCount); + persistor.deleteLogsOlderThan(oldestRetainedTimestamp); + } + } catch (IOException ioe) { + abortService("Snapshot (timestamp " + snapshotTime + ") failed due to: " + ioe.getMessage(), ioe); + } + } + + public synchronized TransactionSnapshot getCurrentState() { + return TransactionSnapshot.copyFrom(System.currentTimeMillis(), readPointer, lastWritePointer, + invalidTxList, inProgress, committingChangeSets, + committedChangeSets); + } + + public synchronized void recoverState() { + try { + TransactionSnapshot lastSnapshot = persistor.getLatestSnapshot(); + // if we failed before a snapshot could complete, we might not have one to restore + if (lastSnapshot != null) { + restoreSnapshot(lastSnapshot); + } + // replay any WALs since the last snapshot + Collection logs = persistor.getLogsSince(lastSnapshotTime); + if (logs != null) { + replayLogs(logs); + } + } catch (IOException e) { + LOG.error("Unable to read back transaction state:", e); + throw Throwables.propagate(e); + } + } + + /** + * Restore the initial in-memory transaction state from a snapshot. + */ + private void restoreSnapshot(TransactionSnapshot snapshot) { + LOG.info("Restoring transaction state from snapshot at " + snapshot.getTimestamp()); + Preconditions.checkState(lastSnapshotTime == 0, "lastSnapshotTime has been set!"); + Preconditions.checkState(readPointer == 0, "readPointer has been set!"); + Preconditions.checkState(lastWritePointer == 0, "lastWritePointer has been set!"); + Preconditions.checkState(invalidTxList.isEmpty(), "invalid list should be empty!"); + Preconditions.checkState(inProgress.isEmpty(), "inProgress map should be empty!"); + Preconditions.checkState(committingChangeSets.isEmpty(), "committingChangeSets should be empty!"); + Preconditions.checkState(committedChangeSets.isEmpty(), "committedChangeSets should be empty!"); + LOG.info("Restoring snapshot of state: " + snapshot); + + lastSnapshotTime = snapshot.getTimestamp(); + readPointer = snapshot.getReadPointer(); + lastWritePointer = snapshot.getWritePointer(); + invalidTxList.addAll(snapshot.getInvalid()); + inProgress.putAll(txnBackwardsCompatCheck(defaultLongTimeout, longTimeoutTolerance, snapshot.getInProgress())); + for (Map.Entry> entry : snapshot.getCommittingChangeSets().entrySet()) { + committingChangeSets.put(entry.getKey(), new ChangeSet(null, entry.getValue())); + } + for (Map.Entry> entry : snapshot.getCommittedChangeSets().entrySet()) { + committedChangeSets.put(entry.getKey(), new ChangeSet(null, entry.getValue())); + } + } + + /** + * Check if in-progress transactions need to be migrated to have expiration time and type, if so do the migration. + * This is required for backwards compatibility, when long running transactions were represented + * with expiration time -1. This can be removed when we stop supporting SnapshotCodec version 1. + */ + public static Map txnBackwardsCompatCheck(int defaultLongTimeout, long longTimeoutTolerance, + Map inProgress) { + for (Map.Entry entry : inProgress.entrySet()) { + long writePointer = entry.getKey(); + long expiration = entry.getValue().getExpiration(); + // LONG transactions will either have a negative expiration or expiration set to the long timeout + // use a fudge factor on the expiration check, since expiraton is set based on system time, not the write pointer + if (entry.getValue().getType() == null && + (expiration < 0 || + (getTxExpirationFromWritePointer(writePointer, defaultLongTimeout) - expiration + < longTimeoutTolerance))) { + // handle null expiration + long newExpiration = getTxExpirationFromWritePointer(writePointer, defaultLongTimeout); + InProgressTx compatTx = + new InProgressTx(entry.getValue().getVisibilityUpperBound(), newExpiration, InProgressType.LONG, + entry.getValue().getCheckpointWritePointers()); + entry.setValue(compatTx); + } else if (entry.getValue().getType() == null) { + InProgressTx compatTx = + new InProgressTx(entry.getValue().getVisibilityUpperBound(), entry.getValue().getExpiration(), + InProgressType.SHORT, entry.getValue().getCheckpointWritePointers()); + entry.setValue(compatTx); + } + } + return inProgress; + } + + /** + * Resets the state of the transaction manager. + */ + public void resetState() { + this.logWriteLock.lock(); + try { + // Take a snapshot before resetting the state, for debugging purposes + doSnapshot(false); + // Clear the state + clear(); + // Take another snapshot: if no snapshot is taken after clearing the state + // and the manager is restarted, we will recover from the snapshot taken + // before resetting the state, which would be really bad + // This call will also init a new WAL + doSnapshot(false); + } catch (IOException e) { + LOG.error("Snapshot failed when resetting state!", e); + e.printStackTrace(); + } finally { + this.logWriteLock.unlock(); + } + } + + /** + * Replay all logged edits from the given transaction logs. + */ + private void replayLogs(Collection logs) { + for (TransactionLog log : logs) { + LOG.info("Replaying edits from transaction log " + log.getName()); + int editCnt = 0; + try { + TransactionLogReader reader = log.getReader(); + // reader may be null in the case of an empty file + if (reader == null) { + continue; + } + TransactionEdit edit; + while ((edit = reader.next()) != null) { + editCnt++; + switch (edit.getState()) { + case INPROGRESS: + long expiration = edit.getExpiration(); + TransactionType type = edit.getType(); + // Check if transaction needs to be migrated to have expiration and type. Previous version of + // long running transactions were represented with expiration time as -1. + // This can be removed when we stop supporting TransactionEditCodecV2. + if (expiration < 0) { + expiration = getTxExpirationFromWritePointer(edit.getWritePointer(), defaultLongTimeout); + type = TransactionType.LONG; + } else if (type == null) { + type = TransactionType.SHORT; + } + // We don't persist the client id. + addInProgressAndAdvance(edit.getWritePointer(), edit.getVisibilityUpperBound(), expiration, type, null); + break; + case COMMITTING: + addCommittingChangeSet(edit.getWritePointer(), null, edit.getChanges()); + break; + case COMMITTED: + // TODO: need to reconcile usage of transaction id v/s write pointer TEPHRA-140 + long transactionId = edit.getWritePointer(); + long[] checkpointPointers = edit.getCheckpointPointers(); + long writePointer = checkpointPointers == null || checkpointPointers.length == 0 ? + transactionId : checkpointPointers[checkpointPointers.length - 1]; + doCommit(transactionId, writePointer, new ChangeSet(null, edit.getChanges()), + edit.getCommitPointer(), edit.getCanCommit()); + break; + case INVALID: + doInvalidate(edit.getWritePointer()); + break; + case ABORTED: + type = edit.getType(); + // Check if transaction edit needs to be migrated to have type. Previous versions of + // ABORTED edits did not contain type. + // This can be removed when we stop supporting TransactionEditCodecV2. + if (type == null) { + InProgressTx inProgressTx = inProgress.get(edit.getWritePointer()); + if (inProgressTx != null) { + InProgressType inProgressType = inProgressTx.getType(); + if (InProgressType.CHECKPOINT.equals(inProgressType)) { + // this should never happen, because checkpoints never go into the log edits; + LOG.debug("Ignoring ABORTED edit for a checkpoint transaction {}", edit.getWritePointer()); + break; + } + if (inProgressType != null) { + type = inProgressType.getTransactionType(); + } + } else { + // If transaction is not in-progress, then it has either been already aborted or invalidated. + // We cannot determine the transaction's state based on current information, to be safe invalidate it. + LOG.warn("Invalidating transaction {} as it's type cannot be determined during replay", + edit.getWritePointer()); + doInvalidate(edit.getWritePointer()); + break; + } + } + doAbort(edit.getWritePointer(), edit.getCheckpointPointers(), type); + break; + case TRUNCATE_INVALID_TX: + if (edit.getTruncateInvalidTxTime() != 0) { + doTruncateInvalidTxBefore(edit.getTruncateInvalidTxTime()); + } else { + doTruncateInvalidTx(edit.getTruncateInvalidTx()); + } + break; + case CHECKPOINT: + doCheckpoint(edit.getWritePointer(), edit.getParentWritePointer()); + break; + default: + // unknown type! + throw new IllegalArgumentException("Invalid state for WAL entry: " + edit.getState()); + } + } + } catch (IOException | InvalidTruncateTimeException e) { + throw Throwables.propagate(e); + } + LOG.info("Read " + editCnt + " edits from log " + log.getName()); + } + } + + @Override + public void doStop() { + Stopwatch timer = Stopwatch.createStarted(); + LOG.info("Shutting down gracefully..."); + // signal the cleanup thread to stop + if (cleanupThread != null) { + cleanupThread.shutdown(); + try { + cleanupThread.join(30000L); + } catch (InterruptedException ie) { + LOG.warn("Interrupted waiting for cleanup thread to stop"); + Thread.currentThread().interrupt(); + } + } + if (metricsThread != null) { + metricsThread.shutdown(); + try { + metricsThread.join(30000L); + } catch (InterruptedException ie) { + LOG.warn("Interrupted waiting for cleanup thread to stop"); + Thread.currentThread().interrupt(); + } + } + if (snapshotThread != null) { + // this will trigger a final snapshot on stop + snapshotThread.shutdown(); + try { + snapshotThread.join(30000L); + } catch (InterruptedException ie) { + LOG.warn("Interrupted waiting for snapshot thread to stop"); + Thread.currentThread().interrupt(); + } + } + + persistor.stopAsync().awaitTerminated(); + txMetricsCollector.stopAsync().awaitTerminated(); + timer.stop(); + LOG.info("Took " + timer + " to stop"); + notifyStopped(); + } + + /** + * Immediately shuts down the service, without going through the normal close process. + * @param message A message describing the source of the failure. + * @param error Any exception that caused the failure. + */ + private void abortService(String message, Throwable error) { + if (isRunning()) { + LOG.error("Aborting transaction manager due to: " + message, error); + notifyFailed(error); + } + } + + private void ensureAvailable() { + Preconditions.checkState(isRunning(), "Transaction Manager is not running."); + } + + /** + * Start a short transaction with the default timeout. + */ + public Transaction startShort() { + return startShort(defaultTimeout); + } + + /** + * Start a short transaction with a client id and default timeout. + * @param clientId id of the client requesting a transaction. + */ + public Transaction startShort(String clientId) { + return startShort(clientId, defaultTimeout); + } + + /** + * Start a short transaction with a given timeout. + * @param timeoutInSeconds the time out period in seconds. + */ + public Transaction startShort(int timeoutInSeconds) { + return startShort(DEFAULT_CLIENTID, timeoutInSeconds); + } + + /** + * Start a short transaction with a given timeout. + * @param clientId id of the client requesting a transaction. + * @param timeoutInSeconds the time out period in seconds. + */ + public Transaction startShort(String clientId, int timeoutInSeconds) { + Preconditions.checkArgument(clientId != null, "clientId must not be null"); + Preconditions.checkArgument(timeoutInSeconds > 0, + "timeout must be positive but is %s seconds", timeoutInSeconds); + Preconditions.checkArgument(timeoutInSeconds <= maxTimeout, + "timeout must not exceed %s seconds but is %s seconds", maxTimeout, timeoutInSeconds); + txMetricsCollector.rate("start.short"); + Stopwatch timer = Stopwatch.createStarted(); + long expiration = getTxExpiration(timeoutInSeconds); + Transaction tx = startTx(expiration, TransactionType.SHORT, clientId); + txMetricsCollector.histogram("start.short.latency", (int) timer.elapsed(TimeUnit.MILLISECONDS)); + return tx; + } + + private static long getTxExpiration(long timeoutInSeconds) { + long currentTime = System.currentTimeMillis(); + return currentTime + TimeUnit.SECONDS.toMillis(timeoutInSeconds); + } + + public static long getTxExpirationFromWritePointer(long writePointer, long timeoutInSeconds) { + return writePointer / TxConstants.MAX_TX_PER_MS + TimeUnit.SECONDS.toMillis(timeoutInSeconds); + } + + private long getNextWritePointer() { + // We want to align tx ids with current time. We assume that tx ids are sequential, but not less than + // System.currentTimeMillis() * MAX_TX_PER_MS. + return Math.max(lastWritePointer + 1, System.currentTimeMillis() * TxConstants.MAX_TX_PER_MS); + } + + /** + * Start a long transaction. Long transactions and do not participate in conflict detection. Also, aborting a long + * transaction moves it to the invalid list because we assume that its writes cannot be rolled back. + */ + public Transaction startLong() { + return startLong(DEFAULT_CLIENTID); + } + + /** + * Starts a long transaction with a client id. + */ + public Transaction startLong(String clientId) { + Preconditions.checkArgument(clientId != null, "clientId must not be null"); + txMetricsCollector.rate("start.long"); + Stopwatch timer = Stopwatch.createStarted(); + long expiration = getTxExpiration(defaultLongTimeout); + Transaction tx = startTx(expiration, TransactionType.LONG, clientId); + txMetricsCollector.histogram("start.long.latency", (int) timer.elapsed(TimeUnit.MILLISECONDS)); + return tx; + } + + private Transaction startTx(long expiration, TransactionType type, @Nullable String clientId) { + Transaction tx; + long txid; + // guard against changes to the transaction log while processing + this.logReadLock.lock(); + try { + synchronized (this) { + ensureAvailable(); + txid = getNextWritePointer(); + tx = createTransaction(txid, type); + addInProgressAndAdvance(tx.getTransactionId(), tx.getVisibilityUpperBound(), expiration, type, + retainClientId ? clientId : null); + } + // appending to WAL out of global lock for concurrent performance + // we should still be able to arrive at the same state even if log entries are out of order + appendToLog(TransactionEdit.createStarted(tx.getTransactionId(), tx.getVisibilityUpperBound(), expiration, type)); + } finally { + this.logReadLock.unlock(); + } + return tx; + } + + private void addInProgressAndAdvance(long writePointer, long visibilityUpperBound, + long expiration, TransactionType type, @Nullable String clientId) { + addInProgressAndAdvance(writePointer, visibilityUpperBound, expiration, InProgressType.of(type), clientId); + } + + private void addInProgressAndAdvance(long writePointer, long visibilityUpperBound, + long expiration, InProgressType type, @Nullable String clientId) { + inProgress.put(writePointer, new InProgressTx(clientId, visibilityUpperBound, expiration, type)); + advanceWritePointer(writePointer); + } + + private void advanceWritePointer(long writePointer) { + // don't move the write pointer back if we have out of order transaction log entries + if (writePointer > lastWritePointer) { + lastWritePointer = writePointer; + } + } + + public void canCommit(long txId, Collection changeIds) + throws TransactionNotInProgressException, TransactionSizeException, TransactionConflictException { + + txMetricsCollector.rate("canCommit"); + Stopwatch timer = Stopwatch.createStarted(); + InProgressTx inProgressTx = inProgress.get(txId); + if (inProgressTx == null) { + synchronized (this) { + // invalid transaction, either this has timed out and moved to invalid, or something else is wrong. + if (invalidTxList.contains(txId)) { + throw new TransactionNotInProgressException( + String.format( + "canCommit() is called for transaction %d that is not in progress (it is known to be invalid)", txId)); + } else { + throw new TransactionNotInProgressException( + String.format("canCommit() is called for transaction %d that is not in progress", txId)); + } + } + } + + Set set = + validateChangeSet(txId, changeIds, inProgressTx.clientId != null ? inProgressTx.clientId : DEFAULT_CLIENTID); + checkForConflicts(txId, set); + + // guard against changes to the transaction log while processing + this.logReadLock.lock(); + try { + synchronized (this) { + ensureAvailable(); + addCommittingChangeSet(txId, inProgressTx.getClientId(), set); + } + appendToLog(TransactionEdit.createCommitting(txId, set)); + } finally { + this.logReadLock.unlock(); + } + txMetricsCollector.histogram("canCommit.latency", (int) timer.elapsed(TimeUnit.MILLISECONDS)); + } + + /** + * Validate the number of changes and the total size of changes. Log a warning if either of them exceeds the + * configured threshold, or log a warning and throw an exception if it exceeds the configured limit. + * + * We log here because application developers may ignore warnings. Logging here gives us a single point + * (the tx manager log) to identify all clients that send excessively large change sets. + * + * @return the same set of changes, transformed into a set of {@link ChangeId}s. + * @throws TransactionSizeException if the number or total size of the changes exceed the limit. + */ + private Set validateChangeSet(long txId, Collection changeIds, + String clientId) throws TransactionSizeException { + if (changeIds.size() > changeSetCountLimit) { + LOG.warn("Change set for transaction {} belonging to client '{}' has {} entries and exceeds " + + "the allowed size of {}. Limit the number of changes, or use a long-running transaction. ", + txId, clientId, changeIds.size(), changeSetCountLimit); + throw new TransactionSizeException(String.format( + "Change set for transaction %d has %d entries and exceeds the limit of %d", + txId, changeIds.size(), changeSetCountLimit)); + } else if (changeIds.size() > changeSetCountThreshold) { + LOG.warn("Change set for transaction {} belonging to client '{}' has {} entries. " + + "It is recommended to limit the number of changes to {}, or to use a long-running transaction. ", + txId, clientId, changeIds.size(), changeSetCountThreshold); + } + long byteCount = 0L; + Set set = Sets.newHashSetWithExpectedSize(changeIds.size()); + for (byte[] change : changeIds) { + set.add(new ChangeId(change)); + byteCount += change.length; + } + if (byteCount > changeSetSizeLimit) { + LOG.warn("Change set for transaction {} belonging to client '{}' has total size of {} bytes and exceeds " + + "the allowed size of {} bytes. Limit the total size of changes, or use a long-running transaction. ", + txId, clientId, byteCount, changeSetSizeLimit); + throw new TransactionSizeException(String.format( + "Change set for transaction %d has total size of %d bytes and exceeds the limit of %d bytes", + txId, byteCount, changeSetSizeLimit)); + } else if (byteCount > changeSetSizeThreshold) { + LOG.warn("Change set for transaction {} belonging to client '{}' has total size of {} bytes. " + + "It is recommended to limit the total size to {} bytes, or to use a long-running transaction. ", + txId, clientId, byteCount, changeSetSizeThreshold); + } + return set; + } + + private void addCommittingChangeSet(long writePointer, String clientId, Set changes) { + committingChangeSets.put(writePointer, new ChangeSet(retainClientIdPastCommit ? clientId : null, changes)); + } + + public void commit(long txId, long writePointer) + throws TransactionNotInProgressException, TransactionConflictException { + + txMetricsCollector.rate("commit"); + Stopwatch timer = Stopwatch.createStarted(); + ChangeSet changeSet; + boolean addToCommitted = true; + long commitPointer; + // guard against changes to the transaction log while processing + this.logReadLock.lock(); + try { + synchronized (this) { + ensureAvailable(); + // we record commits at the first not-yet assigned transaction id to simplify clearing out change sets that + // are no longer visible by any in-progress transactions + commitPointer = lastWritePointer + 1; + if (inProgress.get(txId) == null) { + // invalid transaction, either this has timed out and moved to invalid, or something else is wrong. + if (invalidTxList.contains(txId)) { + throw new TransactionNotInProgressException( + String.format("canCommit() is called for transaction %d that is not in progress " + + "(it is known to be invalid)", txId)); + } else { + throw new TransactionNotInProgressException( + String.format("canCommit() is called for transaction %d that is not in progress", txId)); + } + } + + // these should be atomic + // NOTE: whether we succeed or not we don't need to keep changes in committing state: same tx cannot + // be attempted to commit twice + changeSet = committingChangeSets.remove(txId); + + if (changeSet != null) { + // double-checking if there are conflicts: someone may have committed since canCommit check + checkForConflicts(txId, changeSet.getChangeIds()); + } else { + // no changes + addToCommitted = false; + } + doCommit(txId, writePointer, changeSet, commitPointer, addToCommitted); + } + appendToLog(TransactionEdit.createCommitted(txId, changeSet == null ? null : changeSet.getChangeIds(), + commitPointer, addToCommitted)); + } finally { + this.logReadLock.unlock(); + } + txMetricsCollector.histogram("commit.latency", (int) timer.elapsed(TimeUnit.MILLISECONDS)); + } + + private void doCommit(long transactionId, long writePointer, ChangeSet changes, long commitPointer, + boolean addToCommitted) { + // In case this method is called when loading a previous WAL, we need to remove the tx from these sets + committingChangeSets.remove(transactionId); + if (addToCommitted && !changes.getChangeIds().isEmpty()) { + // No need to add empty changes to the committed change sets, they will never trigger any conflict + + // Record the committed change set with the next writePointer as the commit time. + // NOTE: we use current next writePointer as key for the map, hence we may have multiple txs changesets to be + // stored under one key + ChangeSet committed = committedChangeSets.get(commitPointer); + if (committed != null) { + // NOTE: we modify the new set to prevent concurrent modification exception, as other threads (e.g. in + // canCommit) use it unguarded + changes.getChangeIds().addAll(committed.getChangeIds()); + } + committedChangeSets.put(commitPointer, changes); + } + // remove from in-progress set, so that it does not get excluded in the future + InProgressTx previous = inProgress.remove(transactionId); + if (previous == null) { + // tx was not in progress! perhaps it timed out and is invalid? try to remove it there. + if (invalidTxList.remove(transactionId)) { + LOG.info("Tx invalid list: removed committed tx {}", transactionId); + } + } else { + LongArrayList checkpointPointers = previous.getCheckpointWritePointers(); + if (!checkpointPointers.isEmpty()) { + // adjust the write pointer to be the last checkpoint of the tx and remove all checkpoints from inProgress + writePointer = checkpointPointers.getLong(checkpointPointers.size() - 1); + inProgress.keySet().removeAll(previous.getCheckpointWritePointers()); + } + } + // moving read pointer + moveReadPointerIfNeeded(writePointer); + + // All committed change sets that are smaller than the earliest started transaction can be removed. + // here we ignore transactions that have no timeout, they are long-running and don't participate in + // conflict detection. + // TODO: for efficiency, can we do this once per-log in replayLogs instead of once per edit? + committedChangeSets.headMap(TxUtils.getFirstShortInProgress(inProgress)).clear(); + } + + public void abort(Transaction tx) { + // guard against changes to the transaction log while processing + txMetricsCollector.rate("abort"); + Stopwatch timer = Stopwatch.createStarted(); + this.logReadLock.lock(); + try { + synchronized (this) { + ensureAvailable(); + doAbort(tx.getTransactionId(), tx.getCheckpointWritePointers(), tx.getType()); + } + appendToLog(TransactionEdit.createAborted(tx.getTransactionId(), tx.getType(), tx.getCheckpointWritePointers())); + txMetricsCollector.histogram("abort.latency", (int) timer.elapsed(TimeUnit.MILLISECONDS)); + } finally { + this.logReadLock.unlock(); + } + } + + private void doAbort(long writePointer, long[] checkpointWritePointers, TransactionType type) { + committingChangeSets.remove(writePointer); + + if (type == TransactionType.LONG) { + // Long running transactions cannot be aborted as their change sets are not saved, + // and hence the changes cannot be rolled back. Invalidate the long running transaction instead. + doInvalidate(writePointer); + return; + } + + // makes tx visible (assumes that all operations were rolled back) + // remove from in-progress set, so that it does not get excluded in the future + InProgressTx removed = inProgress.remove(writePointer); + boolean removeInProgressCheckpoints = true; + if (removed == null) { + // tx was not in progress! perhaps it timed out and is invalid? try to remove it there. + if (invalidTxList.remove(writePointer)) { + // the tx and all its children were invalidated: no need to remove them from inProgress + removeInProgressCheckpoints = false; + // remove any invalidated checkpoint pointers + // this will only be present if the parent write pointer was also invalidated + if (checkpointWritePointers != null) { + for (long checkpointWritePointer : checkpointWritePointers) { + invalidTxList.remove(checkpointWritePointer); + } + } + LOG.info("Tx invalid list: removed aborted tx {}", writePointer); + } + } + if (removeInProgressCheckpoints && checkpointWritePointers != null) { + for (long checkpointWritePointer : checkpointWritePointers) { + inProgress.remove(checkpointWritePointer); + } + } + // removed a tx from excludes: must move read pointer + moveReadPointerIfNeeded(writePointer); + } + + public boolean invalidate(long tx) { + // guard against changes to the transaction log while processing + txMetricsCollector.rate("invalidate"); + Stopwatch timer = Stopwatch.createStarted(); + this.logReadLock.lock(); + try { + boolean success; + synchronized (this) { + ensureAvailable(); + success = doInvalidate(tx); + } + appendToLog(TransactionEdit.createInvalid(tx)); + txMetricsCollector.histogram("invalidate.latency", (int) timer.elapsed(TimeUnit.MILLISECONDS)); + return success; + } finally { + this.logReadLock.unlock(); + } + } + + private boolean doInvalidate(long writePointer) { + ChangeSet previousChangeSet = committingChangeSets.remove(writePointer); + // remove from in-progress set, so that it does not get excluded in the future + InProgressTx previous = inProgress.remove(writePointer); + + // This check is to prevent from invalidating committed transactions + if (previous != null || previousChangeSet != null) { + // add tx to invalids + invalidTxList.add(writePointer); + if (previous == null) { + LOG.debug("Invalidating tx {} in committing change sets but not in-progress", writePointer); + } else { + // invalidate any checkpoint write pointers + LongArrayList childWritePointers = previous.getCheckpointWritePointers(); + if (!childWritePointers.isEmpty()) { + invalidTxList.addAll(childWritePointers); + inProgress.keySet().removeAll(childWritePointers); + } + } + + String clientId = DEFAULT_CLIENTID; + if (previous != null && previous.getClientId() != null) { + clientId = previous.getClientId(); + } + LOG.info("Tx invalid list: added tx {} belonging to client '{}' because of invalidate", writePointer, clientId); + if (previous != null && !previous.isLongRunning()) { + // tx was short-running: must move read pointer + moveReadPointerIfNeeded(writePointer); + } + return true; + } + return false; + } + + /** + * Removes the given transaction ids from the invalid list. + * @param invalidTxIds transaction ids + * @return true if invalid list got changed, false otherwise + */ + public boolean truncateInvalidTx(Set invalidTxIds) { + // guard against changes to the transaction log while processing + txMetricsCollector.rate("truncateInvalidTx"); + Stopwatch timer = Stopwatch.createStarted(); + this.logReadLock.lock(); + try { + boolean success; + synchronized (this) { + ensureAvailable(); + success = doTruncateInvalidTx(invalidTxIds); + } + appendToLog(TransactionEdit.createTruncateInvalidTx(invalidTxIds)); + txMetricsCollector.histogram("truncateInvalidTx.latency", (int) timer.elapsed(TimeUnit.MILLISECONDS)); + return success; + } finally { + this.logReadLock.unlock(); + } + } + + private boolean doTruncateInvalidTx(Set toRemove) { + LOG.info("Removing tx ids {} from invalid list", toRemove); + return invalidTxList.removeAll(toRemove); + } + + /** + * Removes all transaction ids started before the given time from invalid list. + * @param time time in milliseconds + * @return true if invalid list got changed, false otherwise + * @throws InvalidTruncateTimeException if there are any in-progress transactions started before given time + */ + public boolean truncateInvalidTxBefore(long time) throws InvalidTruncateTimeException { + // guard against changes to the transaction log while processing + txMetricsCollector.rate("truncateInvalidTxBefore"); + Stopwatch timer = Stopwatch.createStarted(); + this.logReadLock.lock(); + try { + boolean success; + synchronized (this) { + ensureAvailable(); + success = doTruncateInvalidTxBefore(time); + } + appendToLog(TransactionEdit.createTruncateInvalidTxBefore(time)); + txMetricsCollector.histogram("truncateInvalidTxBefore.latency", (int) timer.elapsed(TimeUnit.MILLISECONDS)); + return success; + } finally { + this.logReadLock.unlock(); + } + } + + private boolean doTruncateInvalidTxBefore(long time) throws InvalidTruncateTimeException { + LOG.info("Removing tx ids before {} from invalid list", time); + long truncateWp = time * TxConstants.MAX_TX_PER_MS; + // Check if there any in-progress transactions started earlier than truncate time + if (inProgress.lowerKey(truncateWp) != null) { + throw new InvalidTruncateTimeException("Transactions started earlier than " + time + " are in-progress"); + } + + // Find all invalid transactions earlier than truncateWp + LongSet toTruncate = new LongArraySet(); + LongIterator it = invalidTxList.toRawList().iterator(); + while (it.hasNext()) { + long wp = it.nextLong(); + if (wp < truncateWp) { + toTruncate.add(wp); + } + } + LOG.info("Removing tx ids {} from invalid list", toTruncate); + return invalidTxList.removeAll(toTruncate); + } + + public Transaction checkpoint(Transaction originalTx) throws TransactionNotInProgressException { + txMetricsCollector.rate("checkpoint"); + Stopwatch timer = Stopwatch.createStarted(); + + Transaction checkpointedTx; + long txId = originalTx.getTransactionId(); + long newWritePointer; + // guard against changes to the transaction log while processing + this.logReadLock.lock(); + try { + synchronized (this) { + ensureAvailable(); + // check that the parent tx is in progress + InProgressTx parentTx = inProgress.get(txId); + if (parentTx == null) { + if (invalidTxList.contains(txId)) { + throw new TransactionNotInProgressException( + String.format("Transaction %d is not in progress because it was invalidated", txId)); + } else { + throw new TransactionNotInProgressException( + String.format("Transaction %d is not in progress", txId)); + } + } + newWritePointer = getNextWritePointer(); + doCheckpoint(newWritePointer, txId); + // create a new transaction with the same read snapshot, plus the additional checkpoint write pointer + // the same read snapshot is maintained to + checkpointedTx = new Transaction(originalTx, newWritePointer, + parentTx.getCheckpointWritePointers().toLongArray()); + } + // appending to WAL out of global lock for concurrent performance + // we should still be able to arrive at the same state even if log entries are out of order + appendToLog(TransactionEdit.createCheckpoint(newWritePointer, txId)); + } finally { + this.logReadLock.unlock(); + } + txMetricsCollector.histogram("checkpoint.latency", (int) timer.elapsed(TimeUnit.MILLISECONDS)); + + return checkpointedTx; + } + + private void doCheckpoint(long newWritePointer, long parentWritePointer) { + InProgressTx existingTx = inProgress.get(parentWritePointer); + existingTx.addCheckpointWritePointer(newWritePointer); + addInProgressAndAdvance(newWritePointer, existingTx.getVisibilityUpperBound(), existingTx.getExpiration(), + InProgressType.CHECKPOINT, existingTx.getClientId()); + } + + // hack for exposing important metric + public int getExcludedListSize() { + return getInvalidSize() + inProgress.size(); + } + + /** + * @return the size of invalid list + */ + public synchronized int getInvalidSize() { + return this.invalidTxList.size(); + } + + int getCommittedSize() { + return this.committedChangeSets.size(); + } + + @Nullable + @VisibleForTesting + InProgressTx getInProgress(long transactionId) { + return inProgress.get(transactionId); + } + + private void checkForConflicts(long txId, Set changeIds) throws TransactionConflictException { + if (changeIds.isEmpty()) { + return; + } + + for (Map.Entry committed : committedChangeSets.entrySet()) { + // If commit time is greater than tx read-pointer, + // basically not visible but committed means "tx committed after given tx was started" + if (committed.getKey() > txId) { + ChangeId change = overlap(committed.getValue().getChangeIds(), changeIds); + if (change != null) { + throw new TransactionConflictException(txId, change.toString(), committed.getValue().getClientId()); + } + } + } + } + + /** + * Checks for overlap in two change sets, returns the first common change it finds, or null if no overlap. + */ + @Nullable + private ChangeId overlap(Set a, Set b) { + // iterate over the smaller set, and check for every element in the other set + if (a.size() > b.size()) { + for (ChangeId change : b) { + if (a.contains(change)) { + return change; + } + } + } else { + for (ChangeId change : a) { + if (b.contains(change)) { + return change; + } + } + } + return null; + } + + private void moveReadPointerIfNeeded(long committedWritePointer) { + if (committedWritePointer > readPointer) { + readPointer = committedWritePointer; + } + } + + /** + * Creates a new Transaction. This method only get called from start transaction, which is already + * synchronized. + */ + private Transaction createTransaction(long writePointer, TransactionType type) { + // For holding the first in progress short transaction Id (with timeout >= 0). + long firstShortTx = Transaction.NO_TX_IN_PROGRESS; + LongArrayList inProgressIds = new LongArrayList(inProgress.size()); + for (Map.Entry entry : inProgress.entrySet()) { + long txId = entry.getKey(); + inProgressIds.add(txId); + if (firstShortTx == Transaction.NO_TX_IN_PROGRESS && !entry.getValue().isLongRunning()) { + firstShortTx = txId; + } + } + return new Transaction(readPointer, writePointer, invalidTxList.toSortedArray(), + inProgressIds.toLongArray(), firstShortTx, type); + } + + private void appendToLog(TransactionEdit edit) { + try { + Stopwatch timer = Stopwatch.createStarted(); + currentLog.append(edit); + txMetricsCollector.rate("wal.append.count"); + txMetricsCollector.histogram("wal.append.latency", (int) timer.elapsed(TimeUnit.MILLISECONDS)); + } catch (IOException ioe) { + abortService("Error appending to transaction log", ioe); + } + } + + private void appendToLog(List edits) { + try { + Stopwatch timer = Stopwatch.createStarted(); + currentLog.append(edits); + txMetricsCollector.rate("wal.append.count", edits.size()); + txMetricsCollector.histogram("wal.append.latency", (int) timer.elapsed(TimeUnit.MILLISECONDS)); + } catch (IOException ioe) { + abortService("Error appending to transaction log", ioe); + } + } + + /** + * Called from the tx service every 10 seconds. + * This hack is needed because current metrics system is not flexible when it comes to adding new metrics. + */ + public void logStatistics() { + LOG.info("Transaction Statistics: write pointer = " + lastWritePointer + + ", invalid = " + getInvalidSize() + + ", in progress = " + inProgress.size() + + ", committing = " + committingChangeSets.size() + + ", committed = " + committedChangeSets.size()); + } + + @SuppressWarnings("unused") + @VisibleForTesting + public TransactionStateStorage getTransactionStateStorage() { + return persistor; + } + + private abstract static class DaemonThreadExecutor extends Thread { + private final AtomicBoolean stopped = new AtomicBoolean(false); + + DaemonThreadExecutor(String name) { + super(name); + setDaemon(true); + } + + public void run() { + try { + while (!isInterrupted() && !stopped.get()) { + doRun(); + synchronized (stopped) { + stopped.wait(getSleepMillis()); + } + } + } catch (InterruptedException ie) { + LOG.info("Interrupted thread " + getName()); + } + // perform any final cleanup + onShutdown(); + LOG.info("Exiting thread " + getName()); + } + + public abstract void doRun(); + + protected abstract long getSleepMillis(); + + protected void onShutdown() { + } + + public void shutdown() { + if (stopped.compareAndSet(false, true)) { + synchronized (stopped) { + stopped.notifyAll(); + } + } + } + } + + /** + * Type of in-progress transaction. + */ + public enum InProgressType { + + /** + * Short transactions detect conflicts during commit. + */ + SHORT(TransactionType.SHORT), + + /** + * Long running transactions do not detect conflicts during commit. + */ + LONG(TransactionType.LONG), + + /** + * Check-pointed transactions are recorded as in-progress. + */ + CHECKPOINT(null); + + private final TransactionType transactionType; + + InProgressType(TransactionType transactionType) { + this.transactionType = transactionType; + } + + public static InProgressType of(TransactionType type) { + switch (type) { + case SHORT: return SHORT; + case LONG: return LONG; + default: throw new IllegalArgumentException("Unknown TransactionType " + type); + } + } + + @Nullable + public TransactionType getTransactionType() { + return transactionType; + } + } + + /** + * Represents some of the info on in-progress tx + */ + public static final class InProgressTx { + /** the oldest in progress tx at the time of this tx start */ + private final long visibilityUpperBound; + private final long expiration; + private final InProgressType type; + private final LongArrayList checkpointWritePointers; + // clientId is not part of hashCode computation or equality check since it is not persisted. Once it is persisted + // and restored, we can include it in the above. + private final String clientId; + + public InProgressTx(String clientId, long visibilityUpperBound, long expiration, InProgressType type) { + this(clientId, visibilityUpperBound, expiration, type, new LongArrayList()); + } + + public InProgressTx(long visibilityUpperBound, long expiration, InProgressType type) { + this(visibilityUpperBound, expiration, type, new LongArrayList()); + } + + public InProgressTx(String clientId, long visibilityUpperBound, long expiration, InProgressType type, + LongArrayList checkpointWritePointers) { + this.visibilityUpperBound = visibilityUpperBound; + this.expiration = expiration; + this.type = type; + this.checkpointWritePointers = checkpointWritePointers; + this.clientId = clientId; + } + + public InProgressTx(long visibilityUpperBound, long expiration, InProgressType type, + LongArrayList checkpointWritePointers) { + this.visibilityUpperBound = visibilityUpperBound; + this.expiration = expiration; + this.type = type; + this.checkpointWritePointers = checkpointWritePointers; + this.clientId = null; + } + + // For backwards compatibility when long running txns were represented with -1 expiration + @Deprecated + public InProgressTx(long visibilityUpperBound, long expiration) { + this(visibilityUpperBound, expiration, null); + } + + public long getVisibilityUpperBound() { + return visibilityUpperBound; + } + + public long getExpiration() { + return expiration; + } + + @Nullable + public InProgressType getType() { + return type; + } + + @Nullable + public String getClientId() { + return clientId; + } + + public boolean isLongRunning() { + if (type == null) { + // for backwards compatibility when long running txns were represented with -1 expiration + return expiration == -1; + } + return type == InProgressType.LONG; + } + + public void addCheckpointWritePointer(long checkpointWritePointer) { + checkpointWritePointers.add(checkpointWritePointer); + } + + public LongArrayList getCheckpointWritePointers() { + return checkpointWritePointers; + } + + @Override + public boolean equals(Object o) { + if (o == null || !(o instanceof InProgressTx)) { + return false; + } + + if (this == o) { + return true; + } + + InProgressTx other = (InProgressTx) o; + return Objects.equal(visibilityUpperBound, other.getVisibilityUpperBound()) && + Objects.equal(expiration, other.getExpiration()) && + Objects.equal(type, other.type) && + Objects.equal(checkpointWritePointers, other.checkpointWritePointers); + } + + @Override + public int hashCode() { + return Objects.hashCode(visibilityUpperBound, expiration, type, checkpointWritePointers); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("visibilityUpperBound", visibilityUpperBound) + .add("expiration", expiration) + .add("type", type) + .add("checkpointWritePointers", checkpointWritePointers) + .add("clientId", clientId) + .toString(); + } + } + + /** + * Represents a set of changes from a client. + */ + public static class ChangeSet { + final String clientId; + final Set changeIds; + + ChangeSet(@Nullable String clientId, Set changeIds) { + this.clientId = clientId; + this.changeIds = changeIds; + } + + @Nullable + public String getClientId() { + return clientId; + } + + public Set getChangeIds() { + return changeIds; + } + } +} diff --git a/cdap-data-fabric/src/main/java/org/apache/tephra/inmemory/InMemoryTransactionService.java b/cdap-data-fabric/src/main/java/org/apache/tephra/inmemory/InMemoryTransactionService.java new file mode 100644 index 000000000000..fca847e94edd --- /dev/null +++ b/cdap-data-fabric/src/main/java/org/apache/tephra/inmemory/InMemoryTransactionService.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tephra.inmemory; + +import com.google.common.util.concurrent.AbstractService; +import com.google.inject.Inject; +import com.google.inject.Provider; +import org.apache.hadoop.conf.Configuration; +import org.apache.tephra.TransactionManager; +import org.apache.tephra.TxConstants; +import org.apache.twill.common.Cancellable; +import org.apache.twill.discovery.Discoverable; +import org.apache.twill.discovery.DiscoveryService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.InetSocketAddress; + +/** + * Transaction server that manages transaction data for the Reactor. + *

+ * Transaction server is HA, one can start multiple instances, only one of which is active and will register itself in + * discovery service. + *

+ */ +public class InMemoryTransactionService extends AbstractService { + private static final Logger LOG = LoggerFactory.getLogger(InMemoryTransactionService.class); + + private final DiscoveryService discoveryService; + private final String serviceName; + // this is Provider, so that we can have multiple instances of it (use a new instance after leader election) + protected final Provider txManagerProvider; + private Cancellable cancelDiscovery; + protected TransactionManager txManager; + + // thrift server config + protected final String address; + protected final int port; + protected final int threads; + protected final int ioThreads; + protected final int maxReadBufferBytes; + + @Inject + public InMemoryTransactionService(Configuration conf, DiscoveryService discoveryService, + Provider txManagerProvider) { + + this.discoveryService = discoveryService; + this.txManagerProvider = txManagerProvider; + this.serviceName = conf.get(TxConstants.Service.CFG_DATA_TX_DISCOVERY_SERVICE_NAME, + TxConstants.Service.DEFAULT_DATA_TX_DISCOVERY_SERVICE_NAME); + + address = conf.get(TxConstants.Service.CFG_DATA_TX_BIND_ADDRESS, TxConstants.Service.DEFAULT_DATA_TX_BIND_ADDRESS); + port = conf.getInt(TxConstants.Service.CFG_DATA_TX_BIND_PORT, TxConstants.Service.DEFAULT_DATA_TX_BIND_PORT); + + // Retrieve the number of threads for the service + threads = conf.getInt(TxConstants.Service.CFG_DATA_TX_SERVER_THREADS, + TxConstants.Service.DEFAULT_DATA_TX_SERVER_THREADS); + ioThreads = conf.getInt(TxConstants.Service.CFG_DATA_TX_SERVER_IO_THREADS, + TxConstants.Service.DEFAULT_DATA_TX_SERVER_IO_THREADS); + + maxReadBufferBytes = conf.getInt(TxConstants.Service.CFG_DATA_TX_THRIFT_MAX_READ_BUFFER, + TxConstants.Service.DEFAULT_DATA_TX_THRIFT_MAX_READ_BUFFER); + + LOG.info("Configuring TransactionService" + + ", address: " + address + + ", port: " + port + + ", threads: " + threads + + ", io threads: " + ioThreads + + ", max read buffer (bytes): " + maxReadBufferBytes); + } + + protected void undoRegister() { + if (cancelDiscovery != null) { + cancelDiscovery.cancel(); + } + } + + protected void doRegister() { + cancelDiscovery = discoveryService.register(new Discoverable(serviceName, getAddress())); + } + + protected InetSocketAddress getAddress() { + return new InetSocketAddress(1); + } + + @Override + protected void doStart() { + try { + txManager = txManagerProvider.get(); + txManager.startAsync().awaitRunning(); + doRegister(); + LOG.info("Transaction Thrift service started successfully on " + getAddress()); + notifyStarted(); + } catch (Throwable t) { + LOG.info("Transaction Thrift service didn't start on " + getAddress()); + notifyFailed(t); + } + } + + @Override + protected void doStop() { + undoRegister(); + txManager.stopAsync().awaitTerminated(); + notifyStopped(); + } +} diff --git a/cdap-data-fabric/src/main/java/org/apache/tephra/persist/AbstractTransactionLog.java b/cdap-data-fabric/src/main/java/org/apache/tephra/persist/AbstractTransactionLog.java new file mode 100644 index 000000000000..1bb492ec0ff4 --- /dev/null +++ b/cdap-data-fabric/src/main/java/org/apache/tephra/persist/AbstractTransactionLog.java @@ -0,0 +1,354 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tephra.persist; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Stopwatch; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Writable; +import org.apache.tephra.TxConstants; +import org.apache.tephra.metrics.MetricsCollector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import javax.annotation.Nullable; + +/** + * Common implementation of a transaction log, backed by file reader and writer based storage. Classes extending + * this class, must also implement {@link TransactionLogWriter} and {@link TransactionLogReader}. + * + * It is important to call close() on this class to ensure that all writes are synced and the log files are closed. + */ +public abstract class AbstractTransactionLog implements TransactionLog { + + private static final Logger LOG = LoggerFactory.getLogger(AbstractTransactionLog.class); + + private final AtomicLong logSequence = new AtomicLong(); + private final MetricsCollector metricsCollector; + protected long timestamp; + private volatile boolean initialized; + private volatile boolean closing; + private volatile boolean closed; + private long writtenUpTo = 0L; + private volatile long syncedUpTo = 0L; + private final Queue pendingWrites = new ConcurrentLinkedQueue<>(); + private TransactionLogWriter writer; + + private int countSinceLastSync = 0; + private long positionBeforeWrite = -1L; + private final Stopwatch stopWatch = Stopwatch.createUnstarted(); + + private final long slowAppendThreshold; + + AbstractTransactionLog(long timestamp, MetricsCollector metricsCollector, Configuration conf) { + this.timestamp = timestamp; + this.metricsCollector = metricsCollector; + this.slowAppendThreshold = conf.getLong(TxConstants.TransactionLog.CFG_SLOW_APPEND_THRESHOLD, + TxConstants.TransactionLog.DEFAULT_SLOW_APPEND_THRESHOLD); + } + + /** + * Initializes the log file, opening a file writer. + * + * @throws java.io.IOException If an error is encountered initializing the file writer. + */ + private synchronized void init() throws IOException { + if (initialized) { + return; + } + this.writer = createWriter(); + this.initialized = true; + } + + /** + * Returns a log writer to be used for appending any new {@link TransactionEdit} objects. + */ + protected abstract TransactionLogWriter createWriter() throws IOException; + + @Override + public abstract String getName(); + + @Override + public long getTimestamp() { + return timestamp; + } + + @Override + public void append(TransactionEdit edit) throws IOException { + append(Collections.singletonList(edit)); + } + + @Override + public void append(List edits) throws IOException { + if (closing) { // or closed, which implies closing + throw new IOException("Log " + getName() + " is closing or already closed, cannot append"); + } + if (!initialized) { + init(); + } + // synchronizing here ensures that elements in the queue are ordered by seq number + synchronized (logSequence) { + for (TransactionEdit edit : edits) { + pendingWrites.add(new Entry(new LongWritable(logSequence.getAndIncrement()), edit)); + } + } + // try to sync all pending edits (competing for this with other threads) + sync(); + } + + /** + * Return all pending writes at the time the method is called, or null if no writes are pending. + * + * Note that after this method returns, there can be additional pending writes, + * added concurrently while the existing pending writes are removed. + */ + @Nullable + private Entry[] getPendingWrites() { + synchronized (this) { + if (pendingWrites.isEmpty()) { + return null; + } + Entry[] entriesToSync = new Entry[pendingWrites.size()]; + for (int i = 0; i < entriesToSync.length; i++) { + entriesToSync[i] = pendingWrites.remove(); + } + return entriesToSync; + } + } + + /** + * When multiple threads try to log edits at the same time, they all will call (@link #append} + * followed by {@link #sync()}, concurrently. Hence, it can happen that multiple {@code append()} + * are followed by a single {@code sync}, or vice versa. + * + * We want to record the time and position of the first {@code append()} after a {@code sync()}, + * then measure the time after the next {@code sync()}, and log a warning if it exceeds a threshold. + * Therefore this is called every time before we write the pending list out to the log writer. + * + * See {@link #stopTimer(TransactionLogWriter)}. + * + * @throws IOException if the position of the writer cannot be determined + */ + private void startTimerIfNeeded(TransactionLogWriter writer, int entryCount) throws IOException { + // no sync needed because this is only called within a sync block + if (positionBeforeWrite == -1L) { + positionBeforeWrite = writer.getPosition(); + countSinceLastSync = 0; + stopWatch.reset().start(); + } + countSinceLastSync += entryCount; + } + + /** + * Called by a {@code sync()} after flushing to file system. Issues a warning if the write(s)+sync + * together exceed a threshold. + * + * See {@link #startTimerIfNeeded(TransactionLogWriter, int)}. + * + * @throws IOException if the position of the writer cannot be determined + */ + private void stopTimer(TransactionLogWriter writer) throws IOException { + // this method is only called by a thread if it actually called sync(), inside a sync block + if (positionBeforeWrite != -1L) { // actually it should never be -1, but just in case + stopWatch.stop(); + long elapsed = stopWatch.elapsed(TimeUnit.MILLISECONDS); + long bytesWritten = writer.getPosition() - positionBeforeWrite; + if (elapsed >= slowAppendThreshold) { + LOG.info("Slow append to log {}, took {} ms for {} entr{} and {} bytes.", + getName(), elapsed, countSinceLastSync, countSinceLastSync == 1 ? "y" : "ies", bytesWritten); + } + metricsCollector.histogram("wal.sync.size", countSinceLastSync); + metricsCollector.histogram("wal.sync.bytes", (int) bytesWritten); // single sync won't exceed max int + } + positionBeforeWrite = -1L; + countSinceLastSync = 0; + } + + private void sync() throws IOException { + // writes out pending entries to the HLog + long latestSeq = 0; + int entryCount = 0; + synchronized (this) { + if (closed) { + if (pendingWrites.isEmpty()) { + // this expected: close() sets closed to true after syncing all pending writes (including ours) + return; + } + // this should never happen because close() only sets closed=true after syncing. + // but if it should happen, we must fail this call because we don't know whether the edit was persisted + throw new IOException( + "Unexpected state: Writer is closed but there are pending edits. Cannot guarantee that edits were persisted"); + } + Entry[] currentPending = getPendingWrites(); + if (currentPending != null) { + entryCount = currentPending.length; + startTimerIfNeeded(writer, entryCount); + writer.commitMarker(entryCount); + for (Entry e : currentPending) { + writer.append(e); + } + // sequence are guaranteed to be ascending, so the last one is the greatest + latestSeq = currentPending[currentPending.length - 1].getKey().get(); + writtenUpTo = latestSeq; + } + } + + // giving up the sync lock here allows other threads to write their edits before the sync happens. + // hence, we can have the edits from n threads in one sync. + + // someone else might have already synced our edits, avoid double syncing + // Note: latestSeq is a local variable and syncedUpTo is volatile; hence this is safe without synchronization + if (syncedUpTo >= latestSeq) { + return; + } + synchronized (this) { + // check again - someone else might have synced our edits while we were waiting to synchronize + if (syncedUpTo >= latestSeq) { + return; + } + if (closed) { + // this should never happen because close() only sets closed=true after syncing. + // but if it should happen, we must fail this call because we don't know whether the edit was persisted + throw new IOException(String.format( + "Unexpected state: Writer is closed but there are unsynced edits up to sequence id %d, and writes have " + + "been synced up to sequence id %d. Cannot guarantee that edits are persisted.", latestSeq, syncedUpTo)); + } + writer.sync(); + syncedUpTo = writtenUpTo; + stopTimer(writer); + } + } + + @Override + public synchronized void close() throws IOException { + if (closed) { + return; + } + // prevent other threads from adding more edits to the pending queue + closing = true; + + // perform a final sync if any outstanding writes + if (!pendingWrites.isEmpty()) { + sync(); + } + // NOTE: writer is lazy-inited, so it can be null + if (writer != null) { + this.writer.close(); + } + this.closed = true; + } + + public boolean isClosed() { + return closed; + } + + @Override + public abstract TransactionLogReader getReader() throws IOException; + + /** + * Represents an entry in the transaction log. Each entry consists of a key, generated from an incrementing sequence + * number, and a value, the {@link TransactionEdit} being stored. + */ + public static class Entry implements Writable { + private LongWritable key; + private TransactionEdit edit; + + // for Writable + public Entry() { + this.key = new LongWritable(); + this.edit = new TransactionEdit(); + } + + public Entry(LongWritable key, TransactionEdit edit) { + this.key = key; + this.edit = edit; + } + + public LongWritable getKey() { + return this.key; + } + + public TransactionEdit getEdit() { + return this.edit; + } + + @Override + public void write(DataOutput out) throws IOException { + this.key.write(out); + this.edit.write(out); + } + + @Override + public void readFields(DataInput in) throws IOException { + this.key.readFields(in); + this.edit.readFields(in); + } + } + + // package private for testing + @SuppressWarnings("deprecation") + @Deprecated + @VisibleForTesting + static class CaskEntry implements Writable { + private LongWritable key; + private co.cask.tephra.persist.TransactionEdit edit; + + // for Writable + @SuppressWarnings("unused") + public CaskEntry() { + this.key = new LongWritable(); + this.edit = new co.cask.tephra.persist.TransactionEdit(); + } + + CaskEntry(LongWritable key, co.cask.tephra.persist.TransactionEdit edit) { + this.key = key; + this.edit = edit; + } + + public LongWritable getKey() { + return this.key; + } + + public co.cask.tephra.persist.TransactionEdit getEdit() { + return this.edit; + } + + @Override + public void write(DataOutput out) throws IOException { + this.key.write(out); + this.edit.write(out); + } + + @Override + public void readFields(DataInput in) throws IOException { + this.key.readFields(in); + this.edit.readFields(in); + } + } +} diff --git a/cdap-data-fabric/src/main/java/org/apache/tephra/persist/LocalFileTransactionStateStorage.java b/cdap-data-fabric/src/main/java/org/apache/tephra/persist/LocalFileTransactionStateStorage.java new file mode 100644 index 000000000000..b455b5e28039 --- /dev/null +++ b/cdap-data-fabric/src/main/java/org/apache/tephra/persist/LocalFileTransactionStateStorage.java @@ -0,0 +1,348 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tephra.persist; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.io.Closeables; +import com.google.common.io.Files; +import com.google.common.primitives.Longs; +import com.google.inject.Inject; +import org.apache.hadoop.conf.Configuration; +import org.apache.tephra.TxConstants; +import org.apache.tephra.metrics.MetricsCollector; +import org.apache.tephra.snapshot.SnapshotCodecProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import javax.annotation.Nullable; + +/** + * Persists transaction snapshots and write-ahead logs to files on the local filesystem. + */ +public class LocalFileTransactionStateStorage extends AbstractTransactionStateStorage { + private static final String TMP_SNAPSHOT_FILE_PREFIX = ".in-progress."; + private static final String SNAPSHOT_FILE_PREFIX = "snapshot."; + private static final String LOG_FILE_PREFIX = "txlog."; + private static final Logger LOG = LoggerFactory.getLogger(LocalFileTransactionStateStorage.class); + static final int BUFFER_SIZE = 16384; + + private static final FilenameFilter SNAPSHOT_FILE_FILTER = new FilenameFilter() { + @Override + public boolean accept(File file, String s) { + return s.startsWith(SNAPSHOT_FILE_PREFIX); + } + }; + + private final String configuredSnapshotDir; + private final Configuration conf; + private final MetricsCollector metricsCollector; + private File snapshotDir; + + @Inject + public LocalFileTransactionStateStorage(Configuration conf, SnapshotCodecProvider codecProvider, + MetricsCollector metricsCollector) { + super(codecProvider); + this.configuredSnapshotDir = conf.get(TxConstants.Manager.CFG_TX_SNAPSHOT_LOCAL_DIR); + this.conf = conf; + this.metricsCollector = metricsCollector; + } + + @Override + protected void startUp() throws Exception { + Preconditions.checkState(configuredSnapshotDir != null, + "Snapshot directory is not configured. Please set " + TxConstants.Manager.CFG_TX_SNAPSHOT_LOCAL_DIR + + " in configuration."); + snapshotDir = new File(configuredSnapshotDir); + } + + @Override + protected void shutDown() throws Exception { + // nothing to do + } + + @Override + public String getLocation() { + return snapshotDir.getAbsolutePath(); + } + + @Override + public void writeSnapshot(TransactionSnapshot snapshot) throws IOException { + // save the snapshot to a temporary file + File snapshotTmpFile = new File(snapshotDir, TMP_SNAPSHOT_FILE_PREFIX + snapshot.getTimestamp()); + LOG.debug("Writing snapshot to temporary file {}", snapshotTmpFile); + OutputStream out = new java.io.FileOutputStream(snapshotTmpFile); + boolean threw = true; + try { + codecProvider.encode(out, snapshot); + threw = false; + } finally { + Closeables.close(out, threw); + } + + // move the temporary file into place with the correct filename + File finalFile = new File(snapshotDir, SNAPSHOT_FILE_PREFIX + snapshot.getTimestamp()); + if (!snapshotTmpFile.renameTo(finalFile)) { + throw new IOException("Failed renaming temporary snapshot file " + snapshotTmpFile.getName() + " to " + + finalFile.getName()); + } + + LOG.debug("Completed snapshot to file {}", finalFile); + } + + @Override + public TransactionSnapshot getLatestSnapshot() throws IOException { + InputStream is = getLatestSnapshotInputStream(); + if (is == null) { + return null; + } + try { + return readSnapshotFile(is); + } finally { + is.close(); + } + } + + @Override + public TransactionVisibilityState getLatestTransactionVisibilityState() throws IOException { + InputStream is = getLatestSnapshotInputStream(); + if (is == null) { + return null; + } + try { + return codecProvider.decodeTransactionVisibilityState(is); + } finally { + is.close(); + } + } + + private InputStream getLatestSnapshotInputStream() throws IOException { + File[] snapshotFiles = snapshotDir.listFiles(SNAPSHOT_FILE_FILTER); + TimestampedFilename mostRecent = null; + for (File file : snapshotFiles) { + TimestampedFilename tsFile = new TimestampedFilename(file); + if (mostRecent == null || tsFile.compareTo(mostRecent) > 0) { + mostRecent = tsFile; + } + } + + if (mostRecent == null) { + LOG.info("No snapshot files found in {}", snapshotDir.getAbsolutePath()); + return null; + } + + return new FileInputStream(mostRecent.getFile()); + } + + private TransactionSnapshot readSnapshotFile(InputStream is) throws IOException { + return codecProvider.decode(is); + } + + @Override + public long deleteOldSnapshots(int numberToKeep) throws IOException { + File[] snapshotFiles = snapshotDir.listFiles(SNAPSHOT_FILE_FILTER); + if (snapshotFiles.length == 0) { + return -1; + } + TimestampedFilename[] snapshotFilenames = new TimestampedFilename[snapshotFiles.length]; + for (int i = 0; i < snapshotFiles.length; i++) { + snapshotFilenames[i] = new TimestampedFilename(snapshotFiles[i]); + } + Arrays.sort(snapshotFilenames, Collections.reverseOrder()); + if (snapshotFilenames.length <= numberToKeep) { + // nothing to delete, just return the oldest timestamp + return snapshotFilenames[snapshotFilenames.length - 1].getTimestamp(); + } + int toRemoveCount = snapshotFilenames.length - numberToKeep; + TimestampedFilename[] toRemove = new TimestampedFilename[toRemoveCount]; + System.arraycopy(snapshotFilenames, numberToKeep, toRemove, 0, toRemoveCount); + int removedCnt = 0; + for (int i = 0; i < toRemove.length; i++) { + File currentFile = toRemove[i].getFile(); + LOG.debug("Removing old snapshot file {}", currentFile.getAbsolutePath()); + if (!toRemove[i].getFile().delete()) { + LOG.error("Failed deleting snapshot file {}", currentFile.getAbsolutePath()); + } else { + removedCnt++; + } + } + long oldestTimestamp = snapshotFilenames[numberToKeep - 1].getTimestamp(); + LOG.info("Removed {} out of {} expected snapshot files older than {}", removedCnt, toRemoveCount, oldestTimestamp); + return oldestTimestamp; + } + + @Override + public List listSnapshots() throws IOException { + File[] snapshots = snapshotDir.listFiles(SNAPSHOT_FILE_FILTER); + return Lists.transform(Arrays.asList(snapshots), new Function() { + @Nullable + @Override + public String apply(@Nullable File input) { + return input.getName(); + } + }); + } + + @Override + public List getLogsSince(long timestamp) throws IOException { + File[] logFiles = snapshotDir.listFiles(new LogFileFilter(timestamp, Long.MAX_VALUE)); + TimestampedFilename[] timestampedFiles = new TimestampedFilename[logFiles.length]; + for (int i = 0; i < logFiles.length; i++) { + timestampedFiles[i] = new TimestampedFilename(logFiles[i]); + } + // logs need to be processed in ascending order + Arrays.sort(timestampedFiles); + return Lists.transform(Arrays.asList(timestampedFiles), new Function() { + @Nullable + @Override + public TransactionLog apply(@Nullable TimestampedFilename input) { + return new LocalFileTransactionLog(input.getFile(), input.getTimestamp(), metricsCollector, conf); + } + }); + } + + @Override + public TransactionLog createLog(long timestamp) throws IOException { + File newLogFile = new File(snapshotDir, LOG_FILE_PREFIX + timestamp); + LOG.info("Creating new transaction log at {}", newLogFile.getAbsolutePath()); + return new LocalFileTransactionLog(newLogFile, timestamp, metricsCollector, conf); + } + + @Override + public void deleteLogsOlderThan(long timestamp) throws IOException { + File[] logFiles = snapshotDir.listFiles(new LogFileFilter(0, timestamp)); + int removedCnt = 0; + for (File file : logFiles) { + LOG.debug("Removing old transaction log {}", file.getPath()); + if (file.delete()) { + removedCnt++; + } else { + LOG.warn("Failed to remove log file {}", file.getAbsolutePath()); + } + } + LOG.debug("Removed {} transaction logs older than {}", removedCnt, timestamp); + } + + @Override + public void setupStorage() throws IOException { + // create the directory if it doesn't exist + if (!snapshotDir.exists()) { + if (!snapshotDir.mkdirs()) { + throw new IOException("Failed to create directory " + configuredSnapshotDir + + " for transaction snapshot storage"); + } + } else { + Preconditions.checkState(snapshotDir.isDirectory(), + "Configured snapshot directory " + configuredSnapshotDir + " is not a directory!"); + Preconditions.checkState(snapshotDir.canWrite(), "Configured snapshot directory " + + configuredSnapshotDir + " exists but is not writable!"); + } + } + + @Override + public List listLogs() throws IOException { + File[] logs = snapshotDir.listFiles(new LogFileFilter(0, Long.MAX_VALUE)); + return Lists.transform(Arrays.asList(logs), new Function() { + @Nullable + @Override + public String apply(@Nullable File input) { + return input.getName(); + } + }); + } + + private static class LogFileFilter implements FilenameFilter { + private final long startTime; + private final long endTime; + + public LogFileFilter(long startTime, long endTime) { + this.startTime = startTime; + this.endTime = endTime; + } + + @Override + public boolean accept(File file, String s) { + if (s.startsWith(LOG_FILE_PREFIX)) { + String[] parts = s.split("\\."); + if (parts.length == 2) { + try { + long fileTime = Long.parseLong(parts[1]); + return fileTime >= startTime && fileTime < endTime; + } catch (NumberFormatException ignored) { + LOG.warn("Filename {} did not match the expected pattern prefix.", s); + } + } + } + return false; + } + } + + /** + * Represents a filename composed of a prefix and a ".timestamp" suffix. This is useful for manipulating both + * snapshot and transaction log filenames. + */ + private static class TimestampedFilename implements Comparable { + private File file; + private String prefix; + private long timestamp; + + public TimestampedFilename(File file) { + this.file = file; + String[] parts = file.getName().split("\\."); + if (parts.length != 2) { + throw new IllegalArgumentException("Filename " + file.getName() + + " did not match the expected pattern prefix.timestamp"); + } + prefix = parts[0]; + timestamp = Long.parseLong(parts[1]); + } + + public File getFile() { + return file; + } + + public String getPrefix() { + return prefix; + } + + public long getTimestamp() { + return timestamp; + } + + @Override + public int compareTo(TimestampedFilename other) { + int res = prefix.compareTo(other.getPrefix()); + if (res == 0) { + res = Longs.compare(timestamp, other.getTimestamp()); + } + return res; + } + } + +} diff --git a/cdap-data-fabric/src/main/java/org/apache/tephra/persist/TransactionEdit.java b/cdap-data-fabric/src/main/java/org/apache/tephra/persist/TransactionEdit.java new file mode 100644 index 000000000000..05581ab5cece --- /dev/null +++ b/cdap-data-fabric/src/main/java/org/apache/tephra/persist/TransactionEdit.java @@ -0,0 +1,376 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tephra.persist; + +import com.google.common.base.MoreObjects; +import com.google.common.base.Objects; +import com.google.common.collect.Sets; +import org.apache.hadoop.io.Writable; +import org.apache.tephra.ChangeId; +import org.apache.tephra.TransactionManager; +import org.apache.tephra.TransactionType; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Set; + +/** + * Represents a transaction state change in the {@link TransactionLog}. + */ +public class TransactionEdit implements Writable { + + /** + * The possible state changes for a transaction. + */ + public enum State { + INPROGRESS, COMMITTING, COMMITTED, INVALID, ABORTED, MOVE_WATERMARK, TRUNCATE_INVALID_TX, CHECKPOINT + } + + private long writePointer; + + /** + * stores the value of visibility upper bound + * (see {@link TransactionManager.InProgressTx#getVisibilityUpperBound()}) + * for edit of {@link State#INPROGRESS} only + */ + private long visibilityUpperBound; + private long commitPointer; + private long expirationDate; + private State state; + private Set changes; + /** Whether or not the COMMITTED change should be fully committed. */ + private boolean canCommit; + private TransactionType type; + private Set truncateInvalidTx; + private long truncateInvalidTxTime; + private long parentWritePointer; + private long[] checkpointPointers; + + // for Writable + public TransactionEdit() { + this.changes = Sets.newHashSet(); + this.truncateInvalidTx = Sets.newHashSet(); + } + + // package private for testing + TransactionEdit(long writePointer, long visibilityUpperBound, State state, long expirationDate, + Set changes, long commitPointer, boolean canCommit, TransactionType type, + Set truncateInvalidTx, long truncateInvalidTxTime, long parentWritePointer, + long[] checkpointPointers) { + this.writePointer = writePointer; + this.visibilityUpperBound = visibilityUpperBound; + this.state = state; + this.expirationDate = expirationDate; + this.changes = changes != null ? changes : Collections.emptySet(); + this.commitPointer = commitPointer; + this.canCommit = canCommit; + this.type = type; + this.truncateInvalidTx = truncateInvalidTx != null ? truncateInvalidTx : Collections.emptySet(); + this.truncateInvalidTxTime = truncateInvalidTxTime; + this.parentWritePointer = parentWritePointer; + this.checkpointPointers = checkpointPointers; + } + + /** + * Returns the transaction write pointer assigned for the state change. + */ + public long getWritePointer() { + return writePointer; + } + + void setWritePointer(long writePointer) { + this.writePointer = writePointer; + } + + public long getVisibilityUpperBound() { + return visibilityUpperBound; + } + + void setVisibilityUpperBound(long visibilityUpperBound) { + this.visibilityUpperBound = visibilityUpperBound; + } + + /** + * Returns the type of state change represented. + */ + public State getState() { + return state; + } + + void setState(State state) { + this.state = state; + } + + /** + * Returns any expiration timestamp (in milliseconds) associated with the state change. This should only + * be populated for changes of type {@link State#INPROGRESS}. + */ + public long getExpiration() { + return expirationDate; + } + + void setExpiration(long expirationDate) { + this.expirationDate = expirationDate; + } + + /** + * @return the set of changed row keys associated with the state change. This is only populated for edits + * of type {@link State#COMMITTING} or {@link State#COMMITTED}. + */ + public Set getChanges() { + return changes; + } + + void setChanges(Set changes) { + this.changes = changes; + } + + /** + * Returns the write pointer used to commit the row key change set. This is only populated for edits of type + * {@link State#COMMITTED}. + */ + public long getCommitPointer() { + return commitPointer; + } + + void setCommitPointer(long commitPointer) { + this.commitPointer = commitPointer; + } + + /** + * Returns whether or not the transaction should be moved to the committed set. This is only populated for edits + * of type {@link State#COMMITTED}. + */ + public boolean getCanCommit() { + return canCommit; + } + + void setCanCommit(boolean canCommit) { + this.canCommit = canCommit; + } + + /** + * Returns the transaction type. This is only populated for edits of type {@link State#INPROGRESS} or + * {@link State#ABORTED}. + */ + public TransactionType getType() { + return type; + } + + void setType(TransactionType type) { + this.type = type; + } + + /** + * Returns the transaction ids to be removed from invalid transaction list. This is only populated for + * edits of type {@link State#TRUNCATE_INVALID_TX} + */ + public Set getTruncateInvalidTx() { + return truncateInvalidTx; + } + + void setTruncateInvalidTx(Set truncateInvalidTx) { + this.truncateInvalidTx = truncateInvalidTx; + } + + /** + * Returns the time until which the invalid transactions need to be truncated from invalid transaction list. + * This is only populated for edits of type {@link State#TRUNCATE_INVALID_TX} + */ + public long getTruncateInvalidTxTime() { + return truncateInvalidTxTime; + } + + void setTruncateInvalidTxTime(long truncateInvalidTxTime) { + this.truncateInvalidTxTime = truncateInvalidTxTime; + } + + /** + * Returns the parent write pointer for a checkpoint operation. This is only populated for edits of type + * {@link State#CHECKPOINT} + */ + public long getParentWritePointer() { + return parentWritePointer; + } + + void setParentWritePointer(long parentWritePointer) { + this.parentWritePointer = parentWritePointer; + } + + /** + * Returns the checkpoint write pointers for the edit. This is only populated for edits of type + * {@link State#ABORTED}. + */ + public long[] getCheckpointPointers() { + return checkpointPointers; + } + + void setCheckpointPointers(long[] checkpointPointers) { + this.checkpointPointers = checkpointPointers; + } + + /** + * Creates a new instance in the {@link State#INPROGRESS} state. + */ + public static TransactionEdit createStarted(long writePointer, long visibilityUpperBound, + long expirationDate, TransactionType type) { + return new TransactionEdit(writePointer, visibilityUpperBound, State.INPROGRESS, + expirationDate, null, 0L, false, type, null, 0L, 0L, null); + } + + /** + * Creates a new instance in the {@link State#COMMITTING} state. + */ + public static TransactionEdit createCommitting(long writePointer, Set changes) { + return new TransactionEdit(writePointer, 0L, State.COMMITTING, 0L, changes, 0L, false, null, null, 0L, 0L, null); + } + + /** + * Creates a new instance in the {@link State#COMMITTED} state. + */ + public static TransactionEdit createCommitted(long writePointer, Set changes, long nextWritePointer, + boolean canCommit) { + return new TransactionEdit(writePointer, 0L, State.COMMITTED, 0L, changes, nextWritePointer, canCommit, null, + null, 0L, 0L, null); + } + + /** + * Creates a new instance in the {@link State#ABORTED} state. + */ + public static TransactionEdit createAborted(long writePointer, TransactionType type, long[] checkpointPointers) { + return new TransactionEdit(writePointer, 0L, State.ABORTED, 0L, null, 0L, false, type, null, 0L, 0L, + checkpointPointers); + } + + /** + * Creates a new instance in the {@link State#INVALID} state. + */ + public static TransactionEdit createInvalid(long writePointer) { + return new TransactionEdit(writePointer, 0L, State.INVALID, 0L, null, 0L, false, null, null, 0L, 0L, null); + } + + /** + * Creates a new instance in the {@link State#MOVE_WATERMARK} state. + */ + public static TransactionEdit createMoveWatermark(long writePointer) { + return new TransactionEdit(writePointer, 0L, State.MOVE_WATERMARK, 0L, null, 0L, false, null, null, 0L, 0L, null); + } + + /** + * Creates a new instance in the {@link State#TRUNCATE_INVALID_TX} state. + */ + public static TransactionEdit createTruncateInvalidTx(Set truncateInvalidTx) { + return new TransactionEdit(0L, 0L, State.TRUNCATE_INVALID_TX, 0L, null, 0L, false, null, truncateInvalidTx, + 0L, 0L, null); + } + + /** + * Creates a new instance in the {@link State#TRUNCATE_INVALID_TX} state. + */ + public static TransactionEdit createTruncateInvalidTxBefore(long truncateInvalidTxTime) { + return new TransactionEdit(0L, 0L, State.TRUNCATE_INVALID_TX, 0L, null, 0L, false, null, null, + truncateInvalidTxTime, 0L, null); + } + + /** + * Creates a new instance in the {@link State#CHECKPOINT} state. + */ + public static TransactionEdit createCheckpoint(long writePointer, long parentWritePointer) { + return new TransactionEdit(writePointer, 0L, State.CHECKPOINT, 0L, null, 0L, false, null, null, 0L, + parentWritePointer, null); + } + + /** + * Creates a new instance from {@link co.cask.tephra.persist.TransactionEdit}. + */ + @Deprecated + public static TransactionEdit convertCaskTxEdit(co.cask.tephra.persist.TransactionEdit txEdit) { + if (txEdit == null) { + return null; + } + return new TransactionEdit(txEdit.getWritePointer(), txEdit.getVisibilityUpperBound(), + TransactionEdit.State.valueOf(txEdit.getState().toString()), txEdit.getExpiration(), + txEdit.getChanges(), txEdit.getCommitPointer(), txEdit.getCanCommit(), txEdit.getType(), + txEdit.getTruncateInvalidTx(), txEdit.getTruncateInvalidTxTime(), + txEdit.getParentWritePointer(), txEdit.getCheckpointPointers()); + } + + @Override + public void write(DataOutput out) throws IOException { + TransactionEditCodecs.encode(this, out); + } + + @Override + public void readFields(DataInput in) throws IOException { + TransactionEditCodecs.decode(this, in); + } + + @Override + public final boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof TransactionEdit)) { + return false; + } + + TransactionEdit that = (TransactionEdit) o; + + return Objects.equal(this.writePointer, that.writePointer) && + Objects.equal(this.visibilityUpperBound, that.visibilityUpperBound) && + Objects.equal(this.commitPointer, that.commitPointer) && + Objects.equal(this.expirationDate, that.expirationDate) && + Objects.equal(this.state, that.state) && + Objects.equal(this.changes, that.changes) && + Objects.equal(this.canCommit, that.canCommit) && + Objects.equal(this.type, that.type) && + Objects.equal(this.truncateInvalidTx, that.truncateInvalidTx) && + Objects.equal(this.truncateInvalidTxTime, that.truncateInvalidTxTime) && + Objects.equal(this.parentWritePointer, that.parentWritePointer) && + Arrays.equals(this.checkpointPointers, that.checkpointPointers); + } + + @Override + public final int hashCode() { + return Objects.hashCode(writePointer, visibilityUpperBound, commitPointer, expirationDate, state, changes, + canCommit, type, parentWritePointer, checkpointPointers); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("writePointer", writePointer) + .add("visibilityUpperBound", visibilityUpperBound) + .add("commitPointer", commitPointer) + .add("expiration", expirationDate) + .add("state", state) + .add("changesSize", changes != null ? changes.size() : 0) + .add("canCommit", canCommit) + .add("type", type) + .add("truncateInvalidTx", truncateInvalidTx) + .add("truncateInvalidTxTime", truncateInvalidTxTime) + .add("parentWritePointer", parentWritePointer) + .add("checkpointPointers", checkpointPointers) + .toString(); + } + +} diff --git a/cdap-data-fabric/src/main/java/org/apache/tephra/persist/TransactionSnapshot.java b/cdap-data-fabric/src/main/java/org/apache/tephra/persist/TransactionSnapshot.java new file mode 100644 index 000000000000..3a2ff96de9c7 --- /dev/null +++ b/cdap-data-fabric/src/main/java/org/apache/tephra/persist/TransactionSnapshot.java @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tephra.persist; + +import com.google.common.base.MoreObjects; +import com.google.common.base.Objects; +import com.google.common.collect.Maps; +import it.unimi.dsi.fastutil.longs.LongArrayList; +import org.apache.tephra.ChangeId; +import org.apache.tephra.TransactionManager; +import org.apache.tephra.manager.InvalidTxList; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Set; +import java.util.TreeMap; + +/** + * Represents an in-memory snapshot of the full transaction state. + */ +public class TransactionSnapshot implements TransactionVisibilityState { + private long timestamp; + private long readPointer; + private long writePointer; + private Collection invalid; + private NavigableMap inProgress; + private Map> committingChangeSets; + private Map> committedChangeSets; + + /** + * Creates an instance of TransactionSnapshot with the given transaction state + * + * @param timestamp timestamp, in millis, that the snapshot was taken + * @param readPointer current transaction read pointer + * @param writePointer current transaction write pointer + * @param invalid current list of invalid write pointer; must be sorted + * @param inProgress current map of in-progress write pointers to expiration timestamps + * @param committing current map of write pointers to change sets which have passed {@code canCommit()} but not + * yet committed + * @param committed current map of write pointers to change sets which have committed + */ + public TransactionSnapshot(long timestamp, long readPointer, long writePointer, Collection invalid, + NavigableMap inProgress, + Map> committing, Map> committed) { + this(timestamp, readPointer, writePointer, invalid, inProgress); + this.committingChangeSets = committing; + this.committedChangeSets = committed; + } + + /** + * Creates an instance of TransactionSnapshot with the given transaction state + * + * @param timestamp timestamp, in millis, that the snapshot was taken + * @param readPointer current transaction read pointer + * @param writePointer current transaction write pointer + * @param invalid current list of invalid write pointer; must be sorted + * @param inProgress current map of in-progress write pointers to expiration timestamps + */ + public TransactionSnapshot(long timestamp, long readPointer, long writePointer, Collection invalid, + NavigableMap inProgress) { + this.timestamp = timestamp; + this.readPointer = readPointer; + this.writePointer = writePointer; + this.invalid = invalid; + this.inProgress = inProgress; + this.committingChangeSets = Collections.emptyMap(); + this.committedChangeSets = Collections.emptyMap(); + } + + @Override + public long getTimestamp() { + return timestamp; + } + + @Override + public long getReadPointer() { + return readPointer; + } + + @Override + public long getWritePointer() { + return writePointer; + } + + @Override + public Collection getInvalid() { + return invalid; + } + + @Override + public NavigableMap getInProgress() { + return inProgress; + } + + @Override + public long getVisibilityUpperBound() { + // the readPointer of the oldest in-progress tx is the oldest in use + // todo: potential problem with not moving visibility upper bound for the whole duration of long-running tx + Map.Entry firstInProgress = inProgress.firstEntry(); + if (firstInProgress == null) { + // using readPointer as smallest visible when non txs are there + return readPointer; + } + return firstInProgress.getValue().getVisibilityUpperBound(); + } + + /** + * Returns a map of transaction write pointer to sets of changed row keys for transactions that had called + * {@code InMemoryTransactionManager.canCommit(Transaction, Collection)} but not yet called + * {@code InMemoryTransactionManager.commit(Transaction)} at the time of the snapshot. + * + * @return a map of transaction write pointer to set of changed row keys. + */ + public Map> getCommittingChangeSets() { + return committingChangeSets; + } + + /** + * Returns a map of transaction write pointer to set of changed row keys for transaction that had successfully called + * {@code InMemoryTransactionManager.commit(Transaction)} at the time of the snapshot. + * + * @return a map of transaction write pointer to set of changed row keys. + */ + public Map> getCommittedChangeSets() { + return committedChangeSets; + } + + /** + * Checks that this instance matches another {@code TransactionSnapshot} instance. Note that the equality check + * ignores the snapshot timestamp value, but includes all other properties. + * + * @param obj the other instance to check for equality. + * @return {@code true} if the instances are equal, {@code false} if not. + */ + @Override + public boolean equals(Object obj) { + if (!(obj instanceof TransactionSnapshot)) { + return false; + } + TransactionSnapshot other = (TransactionSnapshot) obj; + return readPointer == other.readPointer && + writePointer == other.writePointer && + invalid.equals(other.invalid) && + inProgress.equals(other.inProgress) && + committingChangeSets.equals(other.committingChangeSets) && + committedChangeSets.equals(other.committedChangeSets); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("timestamp", timestamp) + .add("readPointer", readPointer) + .add("writePointer", writePointer) + .add("invalidSize", invalid.size()) + .add("inProgressSize", inProgress.size()) + .add("committingSize", committingChangeSets.size()) + .add("committedSize", committedChangeSets.size()) + .toString(); + } + + @Override + public int hashCode() { + return Objects.hashCode(readPointer, writePointer, invalid, inProgress, committingChangeSets, committedChangeSets); + } + + /** + * Creates a new {@code TransactionSnapshot} instance with copies of all of the individual collections. + * @param snapshotTime timestamp, in millis, that the snapshot was taken + * @param readPointer current transaction read pointer + * @param writePointer current transaction write pointer + * @param invalidTxList current list of invalid write pointers + * @param inProgress current map of in-progress write pointers to expiration timestamps + * @param committing current map of write pointers to change sets which have passed {@code canCommit()} but not + * yet committed + * @param committed current map of write pointers to change sets which have committed + * @return a new {@code TransactionSnapshot} instance + */ + public static TransactionSnapshot copyFrom(long snapshotTime, long readPointer, + long writePointer, InvalidTxList invalidTxList, + NavigableMap inProgress, + Map committing, + NavigableMap committed) { + // copy invalid IDs, after sorting + Collection invalidCopy = new LongArrayList(invalidTxList.toSortedArray()); + // copy in-progress IDs and expirations + NavigableMap inProgressCopy = Maps.newTreeMap(inProgress); + + // for committing and committed maps, we need to copy each individual Set as well to prevent modification + Map> committingCopy = Maps.newHashMap(); + for (Map.Entry entry : committing.entrySet()) { + committingCopy.put(entry.getKey(), new HashSet<>(entry.getValue().getChangeIds())); + } + + NavigableMap> committedCopy = new TreeMap<>(); + for (Map.Entry entry : committed.entrySet()) { + committedCopy.put(entry.getKey(), new HashSet<>(entry.getValue().getChangeIds())); + } + + return new TransactionSnapshot(snapshotTime, readPointer, writePointer, + invalidCopy, inProgressCopy, committingCopy, committedCopy); + } + +} diff --git a/cdap-data-fabric/src/test/java/io/cdap/cdap/api/dataset/lib/ReflectionTableTest.java b/cdap-data-fabric/src/test/java/io/cdap/cdap/api/dataset/lib/ReflectionTableTest.java index fa9df5f086a8..5e7c732859e7 100644 --- a/cdap-data-fabric/src/test/java/io/cdap/cdap/api/dataset/lib/ReflectionTableTest.java +++ b/cdap-data-fabric/src/test/java/io/cdap/cdap/api/dataset/lib/ReflectionTableTest.java @@ -144,7 +144,7 @@ public int hashCode() { @Override public String toString() { - return Objects.toStringHelper(this) + return com.google.common.base.MoreObjects.toStringHelper(this) .add("firstName", firstName) .add("lastName", lastName) .add("id", id) diff --git a/cdap-data-fabric/src/test/java/io/cdap/cdap/api/dataset/lib/TimeseriesTableScannerTest.java b/cdap-data-fabric/src/test/java/io/cdap/cdap/api/dataset/lib/TimeseriesTableScannerTest.java index 2aa94e6b77ea..6bb96b2923d5 100644 --- a/cdap-data-fabric/src/test/java/io/cdap/cdap/api/dataset/lib/TimeseriesTableScannerTest.java +++ b/cdap-data-fabric/src/test/java/io/cdap/cdap/api/dataset/lib/TimeseriesTableScannerTest.java @@ -16,7 +16,7 @@ package io.cdap.cdap.api.dataset.lib; -import com.google.common.base.Objects; + import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedMap; @@ -263,7 +263,7 @@ public byte[] buildKey() { @Override public String toString() { - return Objects.toStringHelper(Fact.class) + return com.google.common.base.MoreObjects.toStringHelper(Fact.class) .add("ts", ts) .add("dimensions", dimensions) .toString(); diff --git a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/datafabric/dataset/RemoteDatasetFrameworkTest.java b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/datafabric/dataset/RemoteDatasetFrameworkTest.java index f479f08b2189..0b40e2340765 100644 --- a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/datafabric/dataset/RemoteDatasetFrameworkTest.java +++ b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/datafabric/dataset/RemoteDatasetFrameworkTest.java @@ -131,7 +131,7 @@ protected void configure() { // Tx Manager to support working with datasets txManager = injector.getInstance(TransactionManager.class); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); TransactionRunner transactionRunner = injector.getInstance(TransactionRunner.class); StructuredTableAdmin structuredTableAdmin = injector.getInstance(StructuredTableAdmin.class); StoreDefinition.createAllTables(structuredTableAdmin); @@ -155,7 +155,7 @@ protected void configure() { ImmutableSet.of(new DatasetAdminOpHTTPHandler(datasetAdminService)); opExecutorService = new DatasetOpExecutorService(cConf, SConfiguration.create(), discoveryService, commonNettyHttpServiceFactory, handlers); - opExecutorService.startAndWait(); + opExecutorService.startAsync().awaitRunning(); AccessEnforcer accessEnforcer = injector.getInstance(AccessEnforcer.class); @@ -182,7 +182,7 @@ protected void configure() { new HashSet<>(), typeService, instanceService); // Start dataset service, wait for it to be discoverable - service.startAndWait(); + service.startAsync().awaitRunning(); EndpointStrategy endpointStrategy = new RandomEndpointStrategy( () -> discoveryServiceClient.discover(Constants.Service.DATASET_MANAGER)); Preconditions.checkNotNull(endpointStrategy.pick(5, TimeUnit.SECONDS), diff --git a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/datafabric/dataset/service/DatasetServiceTestBase.java b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/datafabric/dataset/service/DatasetServiceTestBase.java index 5e59b8189f03..008825e21d2a 100644 --- a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/datafabric/dataset/service/DatasetServiceTestBase.java +++ b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/datafabric/dataset/service/DatasetServiceTestBase.java @@ -187,7 +187,7 @@ protected void configure() { dsFramework = injector.getInstance(RemoteDatasetFramework.class); // Tx Manager to support working with datasets txManager = injector.getInstance(TransactionManager.class); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); StructuredTableAdmin structuredTableAdmin = injector.getInstance(StructuredTableAdmin.class); StoreDefinition.createAllTables(structuredTableAdmin); @@ -206,7 +206,7 @@ protected void configure() { opExecutorService = new DatasetOpExecutorService(cConf, SConfiguration.create(), discoveryService, commonNettyHttpServiceFactory, handlers); - opExecutorService.startAndWait(); + opExecutorService.startAsync().awaitRunning(); Map defaultModules = injector.getInstance(Key.get(new TypeLiteral>() { }, @@ -252,7 +252,7 @@ protected void configure() { new HashSet<>(), typeService, instanceService); // Start dataset service, wait for it to be discoverable - service.startAndWait(); + service.startAsync().awaitRunning(); waitForService(Constants.Service.DATASET_EXECUTOR); waitForService(Constants.Service.DATASET_MANAGER); // this usually happens while creating a namespace, however not doing that in data fabric tests diff --git a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/datafabric/dataset/service/executor/DatasetOpExecutorServiceTest.java b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/datafabric/dataset/service/executor/DatasetOpExecutorServiceTest.java index 7558ede43a0b..38c1d6eda357 100644 --- a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/datafabric/dataset/service/executor/DatasetOpExecutorServiceTest.java +++ b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/datafabric/dataset/service/executor/DatasetOpExecutorServiceTest.java @@ -16,7 +16,7 @@ package io.cdap.cdap.data2.datafabric.dataset.service.executor; -import com.google.common.base.Objects; + import com.google.common.collect.ImmutableList; import com.google.gson.Gson; import com.google.inject.AbstractModule; @@ -147,15 +147,15 @@ protected void configure() { }); txManager = injector.getInstance(TransactionManager.class); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class)); dsOpExecService = injector.getInstance(DatasetOpExecutorService.class); - dsOpExecService.startAndWait(); + dsOpExecService.startAsync().awaitRunning(); managerService = injector.getInstance(DatasetService.class); - managerService.startAndWait(); + managerService.startAsync().awaitRunning(); dsFramework = injector.getInstance(DatasetFramework.class); @@ -173,10 +173,10 @@ protected void configure() { public void tearDown() throws Exception { dsFramework = null; - dsOpExecService.stopAndWait(); + dsOpExecService.stopAsync().awaitTerminated(); dsOpExecService = null; - managerService.stopAndWait(); + managerService.stopAsync().awaitTerminated(); managerService = null; namespaceAdmin.delete(NamespaceId.DEFAULT); @@ -269,7 +269,8 @@ private URL resolve(String path) throws MalformedURLException { } private DatasetAdminOpResponse getResponse(byte[] body) { - return Objects.firstNonNull(GSON.fromJson(Bytes.toString(body), DatasetAdminOpResponse.class), + return com.google.common.base.MoreObjects.firstNonNull( + GSON.fromJson(Bytes.toString(body), DatasetAdminOpResponse.class), new DatasetAdminOpResponse(null, null)); } } diff --git a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/DatasetFrameworkTestUtil.java b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/DatasetFrameworkTestUtil.java index ee2d66f00e23..97272b5bae44 100644 --- a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/DatasetFrameworkTestUtil.java +++ b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/DatasetFrameworkTestUtil.java @@ -102,7 +102,7 @@ protected void configure() { ); txManager = injector.getInstance(TransactionManager.class); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); framework = injector.getInstance(DatasetFramework.class); } @@ -110,7 +110,7 @@ protected void configure() { @Override protected void after() { if (txManager != null) { - txManager.stopAndWait(); + txManager.stopAsync().awaitTerminated(); } if (tmpFolder != null) { tmpFolder.delete(); diff --git a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/cube/AbstractCubeTest.java b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/cube/AbstractCubeTest.java index 633440c77618..c5a56cbae98c 100644 --- a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/cube/AbstractCubeTest.java +++ b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/cube/AbstractCubeTest.java @@ -560,8 +560,19 @@ public void testMetricsAggregationOptionSum() throws Exception { null); List result = new ArrayList<>(cube.query(query)); Assert.assertEquals(2, result.size()); - verifySumAggregation(result.get(0), "metric1", 100, 1, 1, 0, 0); - verifySumAggregation(result.get(1), "metric2", 100, 2, 1, 0, 0); + TimeSeries m1Series = null; + TimeSeries m2Series = null; + for (TimeSeries ts : result) { + if ("metric1".equals(ts.getMeasureName())) { + m1Series = ts; + } else if ("metric2".equals(ts.getMeasureName())) { + m2Series = ts; + } + } + Assert.assertNotNull(m1Series); + Assert.assertNotNull(m2Series); + verifySumAggregation(m1Series, "metric1", 100, 1, 1, 0, 0); + verifySumAggregation(m2Series, "metric2", 100, 2, 1, 0, 0); // test aggregation option with sum for metric1 and metric2 for agg1, 5 data points for agg1 should get returned // for both metric1 and metric2 @@ -573,10 +584,21 @@ public void testMetricsAggregationOptionSum() throws Exception { null); result = new ArrayList<>(cube.query(query)); Assert.assertEquals(2, result.size()); + m1Series = null; + m2Series = null; + for (TimeSeries ts : result) { + if ("metric1".equals(ts.getMeasureName())) { + m1Series = ts; + } else if ("metric2".equals(ts.getMeasureName())) { + m2Series = ts; + } + } + Assert.assertNotNull(m1Series); + Assert.assertNotNull(m2Series); // metric1 increment by 1 per second, so sum will be 100/5=20, metric2 increment by 2 per second, so sum will be // 200/5=40 - verifySumAggregation(result.get(0), "metric1", 5, 20, 20, 0, 0); - verifySumAggregation(result.get(1), "metric2", 5, 40, 20, 0, 0); + verifySumAggregation(m1Series, "metric1", 5, 20, 20, 0, 0); + verifySumAggregation(m2Series, "metric2", 5, 40, 20, 0, 0); // test aggregation option with sum for metric1 with tag name dim1, it should return two time series for agg1 and // agg2 for metric1, each with 5 data points @@ -587,10 +609,21 @@ public void testMetricsAggregationOptionSum() throws Exception { null); result = new ArrayList<>(cube.query(query)); Assert.assertEquals(2, result.size()); + TimeSeries agg1Series = null; + TimeSeries agg2Series = null; + for (TimeSeries ts : result) { + if ("tag2".equals(ts.getDimensionValues().get("dim2"))) { + agg1Series = ts; + } else if ("tag4".equals(ts.getDimensionValues().get("dim2"))) { + agg2Series = ts; + } + } + Assert.assertNotNull(agg1Series); + Assert.assertNotNull(agg2Series); // agg1 gets increment by 1 for 100 seconds, so sum will be 100/5=20, agg2 gets increment by 3 for 50 seconds, so // sum will be 3*50/5=30 - verifySumAggregation(result.get(0), "metric1", 5, 30, 10, 0, 0); - verifySumAggregation(result.get(1), "metric1", 5, 20, 20, 0, 0); + verifySumAggregation(agg2Series, "metric1", 5, 30, 10, 0, 0); + verifySumAggregation(agg1Series, "metric1", 5, 20, 20, 0, 0); // test metric1 with count 9, this will have a remainder 100%9=1, so there will be 9 aggregated data points, each // with partition size 11 diff --git a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/cube/CubeDatasetTest.java b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/cube/CubeDatasetTest.java index 5f4c92c3c481..545c13c5ecb5 100644 --- a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/cube/CubeDatasetTest.java +++ b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/cube/CubeDatasetTest.java @@ -102,7 +102,7 @@ public void testTxRetryOnFailure() throws Exception { Configuration txConf = new Configuration(); TransactionManager txManager = new TransactionManager(txConf); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); try { TransactionSystemClient txClient = new InMemoryTxSystemClient(txManager); @@ -141,7 +141,7 @@ public void testTxRetryOnFailure() throws Exception { txClient.commitOrThrow(tx); ((TransactionAware) cube2).postTxCommit(); } finally { - txManager.stopAndWait(); + txManager.stopAsync().awaitTerminated(); } } diff --git a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/table/TableTest.java b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/table/TableTest.java index b45514e571d5..14b84bd59897 100644 --- a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/table/TableTest.java +++ b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/table/TableTest.java @@ -127,7 +127,7 @@ protected DatasetAdmin getTableAdmin(DatasetContext datasetContext, String name) public void before() { Configuration txConf = new Configuration(); TransactionManager txManager = new TransactionManager(txConf); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); txClient = new InMemoryTxSystemClient(txManager); } diff --git a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/transaction/TransactionContextTest.java b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/transaction/TransactionContextTest.java index 5313eb9bdb5a..734462e97e2f 100644 --- a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/transaction/TransactionContextTest.java +++ b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/transaction/TransactionContextTest.java @@ -58,13 +58,13 @@ public class TransactionContextTest { @BeforeClass public static void setup() { txManager = new TransactionManager(new Configuration()); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); txClient = new DummyTxClient(txManager); } @AfterClass public static void finish() { - txManager.stopAndWait(); + txManager.stopAsync().awaitTerminated(); } private TransactionContext newTransactionContext(TransactionAware... txAwares) { diff --git a/cdap-data-fabric/src/test/java/io/cdap/cdap/kafka/KafkaTester.java b/cdap-data-fabric/src/test/java/io/cdap/cdap/kafka/KafkaTester.java index dfef63007771..8427e6467050 100644 --- a/cdap-data-fabric/src/test/java/io/cdap/cdap/kafka/KafkaTester.java +++ b/cdap-data-fabric/src/test/java/io/cdap/cdap/kafka/KafkaTester.java @@ -132,15 +132,15 @@ protected void before() throws Throwable { LOG.info("In memory ZK started on {}", zkServer.getConnectionStr()); kafkaServer = new EmbeddedKafkaServer(generateKafkaConfig()); - kafkaServer.startAndWait(); + kafkaServer.startAsync().awaitRunning(); initializeCconf(); injector = createInjector(); zkClient = injector.getInstance(ZKClientService.class); - zkClient.startAndWait(); + zkClient.startAsync().awaitRunning(); kafkaClient = injector.getInstance(KafkaClientService.class); - kafkaClient.startAndWait(); + kafkaClient.startAsync().awaitRunning(); brokerService = injector.getInstance(BrokerService.class); - brokerService.startAndWait(); + brokerService.startAsync().awaitRunning(); String brokerList = updateKafkaBrokerList(injector.getInstance(CConfiguration.class), brokerService); @@ -151,10 +151,10 @@ protected void before() throws Throwable { @Override protected void after() { - brokerService.stopAndWait(); - kafkaClient.stopAndWait(); - zkClient.stopAndWait(); - kafkaServer.stopAndWait(); + brokerService.stopAsync().awaitTerminated(); + kafkaClient.stopAsync().awaitTerminated(); + zkClient.stopAsync().awaitTerminated(); + kafkaServer.stopAsync().awaitTerminated(); zkServer.stopAndWait(); } diff --git a/cdap-data-fabric/src/test/java/io/cdap/cdap/spi/data/nosql/NoSqlStructuredTableAdminTest.java b/cdap-data-fabric/src/test/java/io/cdap/cdap/spi/data/nosql/NoSqlStructuredTableAdminTest.java index 642696716f39..e700b26da8b7 100644 --- a/cdap-data-fabric/src/test/java/io/cdap/cdap/spi/data/nosql/NoSqlStructuredTableAdminTest.java +++ b/cdap-data-fabric/src/test/java/io/cdap/cdap/spi/data/nosql/NoSqlStructuredTableAdminTest.java @@ -42,7 +42,7 @@ public class NoSqlStructuredTableAdminTest extends StructuredTableAdminTest { public static void beforeClass() throws IOException { Configuration txConf = new Configuration(); txManager = new TransactionManager(txConf); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); CConfiguration cConf = dsFrameworkUtil.getConfiguration(); cConf.set(Constants.Dataset.DATA_STORAGE_IMPLEMENTATION, Constants.Dataset.DATA_STORAGE_NOSQL); @@ -57,7 +57,7 @@ protected StructuredTableAdmin getStructuredTableAdmin() throws Exception { @AfterClass public static void afterClass() { if (txManager != null) { - txManager.stopAndWait(); + txManager.stopAsync().awaitTerminated(); } } } diff --git a/cdap-data-fabric/src/test/java/io/cdap/cdap/spi/data/nosql/NoSqlStructuredTableConcurrencyTest.java b/cdap-data-fabric/src/test/java/io/cdap/cdap/spi/data/nosql/NoSqlStructuredTableConcurrencyTest.java index f7379bf1aa40..18795d8e4818 100644 --- a/cdap-data-fabric/src/test/java/io/cdap/cdap/spi/data/nosql/NoSqlStructuredTableConcurrencyTest.java +++ b/cdap-data-fabric/src/test/java/io/cdap/cdap/spi/data/nosql/NoSqlStructuredTableConcurrencyTest.java @@ -54,7 +54,7 @@ protected TransactionRunner getTransactionRunner() { public static void beforeClass() throws IOException { Configuration txConf = new Configuration(); txManager = new TransactionManager(txConf); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); CConfiguration cConf = dsFrameworkUtil.getConfiguration(); cConf.set(Constants.Dataset.DATA_STORAGE_IMPLEMENTATION, Constants.Dataset.DATA_STORAGE_NOSQL); @@ -65,7 +65,7 @@ public static void beforeClass() throws IOException { @AfterClass public static void afterClass() { if (txManager != null) { - txManager.stopAndWait(); + txManager.stopAsync().awaitTerminated(); } } } diff --git a/cdap-data-fabric/src/test/java/io/cdap/cdap/spi/data/nosql/NoSqlStructuredTableRegistryTest.java b/cdap-data-fabric/src/test/java/io/cdap/cdap/spi/data/nosql/NoSqlStructuredTableRegistryTest.java index 4a807929890a..37da4093318d 100644 --- a/cdap-data-fabric/src/test/java/io/cdap/cdap/spi/data/nosql/NoSqlStructuredTableRegistryTest.java +++ b/cdap-data-fabric/src/test/java/io/cdap/cdap/spi/data/nosql/NoSqlStructuredTableRegistryTest.java @@ -48,13 +48,13 @@ protected StructuredTableRegistry getStructuredTableRegistry() { public static void beforeClass() { Configuration txConf = new Configuration(); txManager = new TransactionManager(txConf); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); } @AfterClass public static void afterClass() { if (txManager != null) { - txManager.stopAndWait(); + txManager.stopAsync().awaitTerminated(); } } } diff --git a/cdap-data-fabric/src/test/java/io/cdap/cdap/spi/data/nosql/NoSqlStructuredTableTest.java b/cdap-data-fabric/src/test/java/io/cdap/cdap/spi/data/nosql/NoSqlStructuredTableTest.java index a11bd0d7f9ff..8723bd198d76 100644 --- a/cdap-data-fabric/src/test/java/io/cdap/cdap/spi/data/nosql/NoSqlStructuredTableTest.java +++ b/cdap-data-fabric/src/test/java/io/cdap/cdap/spi/data/nosql/NoSqlStructuredTableTest.java @@ -76,7 +76,7 @@ protected TransactionRunner getTransactionRunner() { public static void beforeClass() throws IOException { Configuration txConf = new Configuration(); txManager = new TransactionManager(txConf); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); CConfiguration cConf = dsFrameworkUtil.getConfiguration(); cConf.set(Constants.Dataset.DATA_STORAGE_IMPLEMENTATION, Constants.Dataset.DATA_STORAGE_NOSQL); @@ -87,7 +87,7 @@ public static void beforeClass() throws IOException { @AfterClass public static void afterClass() { if (txManager != null) { - txManager.stopAndWait(); + txManager.stopAsync().awaitTerminated(); } } diff --git a/cdap-data-fabric/src/test/java/io/cdap/cdap/spi/metadata/dataset/DatasetMetadataStorageTest.java b/cdap-data-fabric/src/test/java/io/cdap/cdap/spi/metadata/dataset/DatasetMetadataStorageTest.java index 82f5036b2fd3..9305b883027e 100644 --- a/cdap-data-fabric/src/test/java/io/cdap/cdap/spi/metadata/dataset/DatasetMetadataStorageTest.java +++ b/cdap-data-fabric/src/test/java/io/cdap/cdap/spi/metadata/dataset/DatasetMetadataStorageTest.java @@ -25,7 +25,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.google.common.io.Closeables; + import com.google.inject.AbstractModule; import com.google.inject.Guice; import com.google.inject.Injector; @@ -107,7 +107,7 @@ protected void configure() { Injector injector = Guice.createInjector(modules); txManager = injector.getInstance(TransactionManager.class); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); storage = injector.getInstance(DatasetMetadataStorage.class); storage.createIndex(); @@ -116,9 +116,9 @@ protected void configure() { @AfterClass public static void teardown() throws IOException { - txManager.stopAndWait(); + txManager.stopAsync().awaitTerminated(); storage.dropIndex(); - Closeables.closeQuietly(storage); + storage.close(); } @Override diff --git a/cdap-data-fabric/src/test/java/io/cdap/cdap/store/DefaultOwnerStoreTest.java b/cdap-data-fabric/src/test/java/io/cdap/cdap/store/DefaultOwnerStoreTest.java index d7d88b415f50..03344c0d737b 100644 --- a/cdap-data-fabric/src/test/java/io/cdap/cdap/store/DefaultOwnerStoreTest.java +++ b/cdap-data-fabric/src/test/java/io/cdap/cdap/store/DefaultOwnerStoreTest.java @@ -86,7 +86,7 @@ protected void configure() { } ); - injector.getInstance(TransactionManager.class).startAndWait(); + injector.getInstance(TransactionManager.class).startAsync().awaitRunning(); txRunner = injector.getInstance(TransactionRunner.class); StoreDefinition.OwnerStore.create(injector.getInstance(StructuredTableAdmin.class)); diff --git a/cdap-distributions/pom.xml b/cdap-distributions/pom.xml index d669e51cf604..745899f251b2 100644 --- a/cdap-distributions/pom.xml +++ b/cdap-distributions/pom.xml @@ -37,6 +37,20 @@ caskdata/cdap-sandbox + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + diff --git a/cdap-docs-gen/pom.xml b/cdap-docs-gen/pom.xml index 62cd7af2c9c8..758d0dc1f757 100644 --- a/cdap-docs-gen/pom.xml +++ b/cdap-docs-gen/pom.xml @@ -51,4 +51,18 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + diff --git a/cdap-docs-gen/src/main/java/io/cdap/cdap/docgen/cli/GenerateCLIDocsTableCommand.java b/cdap-docs-gen/src/main/java/io/cdap/cdap/docgen/cli/GenerateCLIDocsTableCommand.java index 58538f0d3f0d..1f308aa7aa50 100644 --- a/cdap-docs-gen/src/main/java/io/cdap/cdap/docgen/cli/GenerateCLIDocsTableCommand.java +++ b/cdap-docs-gen/src/main/java/io/cdap/cdap/docgen/cli/GenerateCLIDocsTableCommand.java @@ -18,13 +18,8 @@ import com.google.common.base.Function; import com.google.common.base.Joiner; -import com.google.common.base.Predicates; import com.google.common.base.Splitter; -import com.google.common.base.Suppliers; -import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterators; -import com.google.common.collect.Lists; -import com.google.common.collect.Multimap; import io.cdap.cdap.cli.CommandCategory; import io.cdap.cdap.cli.command.system.HelpCommand; import io.cdap.cdap.cli.util.table.TableRendererConfig; @@ -32,10 +27,12 @@ import io.cdap.common.cli.Command; import io.cdap.common.cli.CommandSet; import java.io.PrintStream; +import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.Iterator; import java.util.List; +import java.util.Map; import javax.annotation.Nullable; /** @@ -44,18 +41,17 @@ public class GenerateCLIDocsTableCommand extends HelpCommand { public GenerateCLIDocsTableCommand(CommandSet commands, TableRendererConfig tableRendererConfig) { - super(Suppliers.>>ofInstance( - ImmutableList.>of(commands)), - tableRendererConfig); + super(() -> Collections.singletonList(commands), tableRendererConfig); } @Override public void execute(Arguments arguments, PrintStream output) throws Exception { - Multimap categorizedCommands = categorizeCommands( - commands.get(), CommandCategory.GENERAL, Predicates.alwaysTrue()); + Map> categorizedCommands = categorizeCommands( + commands.get(), CommandCategory.GENERAL, c -> true); for (CommandCategory category : CommandCategory.values()) { output.printf(" **%s**\n", category.getOriginalName()); - List commandList = Lists.newArrayList(categorizedCommands.get(category.getName())); + List commandList = new ArrayList<>( + categorizedCommands.getOrDefault(category.getName(), Collections.emptyList())); Collections.sort(commandList, new Comparator() { @Override public int compare(Command command, Command command2) { diff --git a/cdap-docs/user-guide/source/pipelines/plugins/pom.xml b/cdap-docs/user-guide/source/pipelines/plugins/pom.xml index ee782162c70c..d1fac6327f1b 100644 --- a/cdap-docs/user-guide/source/pipelines/plugins/pom.xml +++ b/cdap-docs/user-guide/source/pipelines/plugins/pom.xml @@ -871,15 +871,18 @@ org.apache.maven.plugins maven-javadoc-plugin - 2.9.1 + 3.6.3 + none + false - http://download.oracle.com/javase/${jee.version}/docs/api/ + https://docs.oracle.com/javase/${jee.version}/docs/api/ ${project.name} ${project.version} Cask Data, Inc. Licensed under the Apache License, Version 2.0.]]> + false diff --git a/cdap-e2e-tests/pom.xml b/cdap-e2e-tests/pom.xml index 8d14ea0dd779..c1823d0bdad5 100644 --- a/cdap-e2e-tests/pom.xml +++ b/cdap-e2e-tests/pom.xml @@ -34,6 +34,17 @@ ${testSourceLocation} + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + diff --git a/cdap-elastic/pom.xml b/cdap-elastic/pom.xml index b95805e275d0..0f2ad2248cd1 100644 --- a/cdap-elastic/pom.xml +++ b/cdap-elastic/pom.xml @@ -98,6 +98,15 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + org.sonatype.central diff --git a/cdap-elastic/src/main/java/io/cdap/cdap/metadata/elastic/ElasticsearchMetadataStorage.java b/cdap-elastic/src/main/java/io/cdap/cdap/metadata/elastic/ElasticsearchMetadataStorage.java index 2f6e1dd5b9bc..af3daae30e2b 100644 --- a/cdap-elastic/src/main/java/io/cdap/cdap/metadata/elastic/ElasticsearchMetadataStorage.java +++ b/cdap-elastic/src/main/java/io/cdap/cdap/metadata/elastic/ElasticsearchMetadataStorage.java @@ -226,7 +226,11 @@ public String getName() { @Override public void close() { - Closeables.closeQuietly(client); + try { + client.close(); + } catch (IOException e) { + // ignore + } } @Override diff --git a/cdap-elastic/src/test/java/io/cdap/cdap/metadata/elastic/ElasticsearchMetadataStorageTest.java b/cdap-elastic/src/test/java/io/cdap/cdap/metadata/elastic/ElasticsearchMetadataStorageTest.java index 60e63cc8d862..29b0b221d443 100644 --- a/cdap-elastic/src/test/java/io/cdap/cdap/metadata/elastic/ElasticsearchMetadataStorageTest.java +++ b/cdap-elastic/src/test/java/io/cdap/cdap/metadata/elastic/ElasticsearchMetadataStorageTest.java @@ -86,7 +86,7 @@ public static void dropIndex() throws IOException { try { elasticStore.dropIndex(); } finally { - Closeables.closeQuietly(elasticStore); + elasticStore.close(); } } } diff --git a/cdap-encryption-ext-tink/pom.xml b/cdap-encryption-ext-tink/pom.xml index 12a4e8cc6dae..89a1f1af43f5 100644 --- a/cdap-encryption-ext-tink/pom.xml +++ b/cdap-encryption-ext-tink/pom.xml @@ -112,6 +112,20 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + dist @@ -143,7 +157,6 @@ org.apache.maven.plugins maven-jar-plugin - 2.4 jar @@ -151,6 +164,7 @@ ${project.build.directory}/libexec ${project.groupId}.${project.build.finalName} + false jar diff --git a/cdap-error-api/pom.xml b/cdap-error-api/pom.xml index 9dd06e519b0a..deccc7e03830 100644 --- a/cdap-error-api/pom.xml +++ b/cdap-error-api/pom.xml @@ -29,4 +29,18 @@ CDAP Error API jar + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + diff --git a/cdap-event-common-spi/pom.xml b/cdap-event-common-spi/pom.xml index b1329a61600a..57706618f195 100644 --- a/cdap-event-common-spi/pom.xml +++ b/cdap-event-common-spi/pom.xml @@ -38,4 +38,18 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + diff --git a/cdap-event-reader-spi/pom.xml b/cdap-event-reader-spi/pom.xml index d11d891c7e99..fb38eaed4b7f 100644 --- a/cdap-event-reader-spi/pom.xml +++ b/cdap-event-reader-spi/pom.xml @@ -44,4 +44,18 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + diff --git a/cdap-event-writer-spi/pom.xml b/cdap-event-writer-spi/pom.xml index 4f5789081625..b1d14631ef3d 100644 --- a/cdap-event-writer-spi/pom.xml +++ b/cdap-event-writer-spi/pom.xml @@ -55,4 +55,18 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + diff --git a/cdap-features/pom.xml b/cdap-features/pom.xml index e1ee8d8d4549..ae792b503ec9 100644 --- a/cdap-features/pom.xml +++ b/cdap-features/pom.xml @@ -38,4 +38,18 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + diff --git a/cdap-formats/pom.xml b/cdap-formats/pom.xml index 7489c1d27e60..6a98416b769b 100644 --- a/cdap-formats/pom.xml +++ b/cdap-formats/pom.xml @@ -59,4 +59,18 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + diff --git a/cdap-formats/src/test/java/io/cdap/cdap/format/StructuredRecordBuilderTest.java b/cdap-formats/src/test/java/io/cdap/cdap/format/StructuredRecordBuilderTest.java index 554a679fe41c..c8711f274f94 100644 --- a/cdap-formats/src/test/java/io/cdap/cdap/format/StructuredRecordBuilderTest.java +++ b/cdap-formats/src/test/java/io/cdap/cdap/format/StructuredRecordBuilderTest.java @@ -28,6 +28,7 @@ import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; import java.util.Date; import java.util.TimeZone; import org.junit.Assert; @@ -108,27 +109,29 @@ public void testTimeSupport() { Schema.Field.of("name", Schema.of(Schema.Type.STRING)), Schema.Field.of("time", Schema.of(Schema.LogicalType.TIME_MILLIS))); - LocalTime expected = LocalTime.now(); + LocalTime baseTime = LocalTime.now(); + LocalTime expectedMillis = baseTime.truncatedTo(ChronoUnit.MILLIS); StructuredRecord record = StructuredRecord.builder(schema) .set("id", 1) .set("name", "test") - .setTime("time", expected).build(); + .setTime("time", expectedMillis).build(); LocalTime actual = record.getTime("time"); - Assert.assertEquals(expected, actual); + Assert.assertEquals(expectedMillis, actual); schema = Schema.recordOf("test", Schema.Field.of("id", Schema.of(Schema.Type.INT)), Schema.Field.of("name", Schema.of(Schema.Type.STRING)), Schema.Field.of("time", Schema.of(Schema.LogicalType.TIME_MICROS))); + LocalTime expectedMicros = baseTime.truncatedTo(ChronoUnit.MICROS); record = StructuredRecord.builder(schema) .set("id", 1) .set("name", "test") - .setTime("time", expected).build(); + .setTime("time", expectedMicros).build(); actual = record.getTime("time"); - Assert.assertEquals(expected, actual); + Assert.assertEquals(expectedMicros, actual); } @Test diff --git a/cdap-gateway/pom.xml b/cdap-gateway/pom.xml index 5bc5668055f2..cf09c74f5ce6 100644 --- a/cdap-gateway/pom.xml +++ b/cdap-gateway/pom.xml @@ -122,6 +122,20 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 17 + 17 + + + + + dist @@ -133,7 +147,6 @@ org.apache.maven.plugins maven-jar-plugin - 2.4 org.apache.maven.plugins diff --git a/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/NettyRouter.java b/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/NettyRouter.java index 022874607582..e68d60b3ea0c 100644 --- a/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/NettyRouter.java +++ b/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/NettyRouter.java @@ -141,7 +141,7 @@ public Optional getBoundAddress() { protected void startUp() throws Exception { // If internal authorization enforcement is enabled, we avoid re-initialization of the token manager. if (SecurityUtil.isManagedSecurity(cConf) && !SecurityUtil.isInternalAuthEnabled(cConf)) { - tokenValidator.startAndWait(); + tokenValidator.startAsync().awaitRunning(); } ChannelGroup channelGroup = new DefaultChannelGroup(ImmediateEventExecutor.INSTANCE); serverCancellable = startServer(createServerBootstrap(channelGroup), channelGroup); @@ -157,14 +157,14 @@ protected void shutDown() { serverCancellable.cancel(); // If internal authorization enforcement is enabled, we avoid duplicate cleanup of the token manager. if (SecurityUtil.isManagedSecurity(cConf) && !SecurityUtil.isInternalAuthEnabled(cConf)) { - tokenValidator.stopAndWait(); + tokenValidator.stopAsync().awaitTerminated(); } LOG.info("Stopped Netty Router."); } @Override - protected Executor executor(final State state) { + protected Executor executor() { final AtomicInteger id = new AtomicInteger(); return runnable -> { Thread t = new Thread(runnable, String.format("NettyRouter-%d", id.incrementAndGet())); diff --git a/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/RouterMain.java b/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/RouterMain.java index 970a6135fbd8..6fcbd8fcab91 100644 --- a/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/RouterMain.java +++ b/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/RouterMain.java @@ -116,7 +116,7 @@ public void start() throws Exception { + "ZooKeeper quorum settings are correct in " + "cdap-site.xml. Currently configured as: %s", zkClientService.getConnectString())); - router.startAndWait(); + router.startAsync().awaitRunning(); LOG.info("Router started."); } diff --git a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/GatewayTestBase.java b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/GatewayTestBase.java index b23aaca4a0ea..7b0c8f96264d 100644 --- a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/GatewayTestBase.java +++ b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/GatewayTestBase.java @@ -16,7 +16,7 @@ package io.cdap.cdap.gateway; -import com.google.common.io.Closeables; + import com.google.common.util.concurrent.Service; import com.google.gson.Gson; import com.google.gson.JsonObject; @@ -179,38 +179,38 @@ protected void configure() { messagingService = injector.getInstance(MessagingService.class); if (messagingService instanceof Service) { - ((Service) messagingService).startAndWait(); + ((Service) messagingService).startAsync().awaitRunning(); } txService = injector.getInstance(TransactionManager.class); - txService.startAndWait(); + txService.startAsync().awaitRunning(); // Define all StructuredTable before starting any services that need StructuredTable StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class)); metadataStorage = injector.getInstance(MetadataStorage.class); metadataStorage.createIndex(); metadataService = injector.getInstance(MetadataService.class); - metadataService.startAndWait(); + metadataService.startAsync().awaitRunning(); dsOpService = injector.getInstance(DatasetOpExecutorService.class); - dsOpService.startAndWait(); + dsOpService.startAsync().awaitRunning(); datasetService = injector.getInstance(DatasetService.class); - datasetService.startAndWait(); + datasetService.startAsync().awaitRunning(); appFabricServer = injector.getInstance(AppFabricServer.class); - appFabricServer.startAndWait(); + appFabricServer.startAsync().awaitRunning(); appFabricProcessorService = injector.getInstance(AppFabricProcessorService.class); - appFabricProcessorService.startAndWait(); + appFabricProcessorService.startAsync().awaitRunning(); logQueryService = injector.getInstance(LogQueryService.class); - logQueryService.startAndWait(); + logQueryService.startAsync().awaitRunning(); metricsQueryService = injector.getInstance(MetricsQueryService.class); - metricsQueryService.startAndWait(); + metricsQueryService.startAsync().awaitRunning(); metricsCollectionService = injector.getInstance(MetricsCollectionService.class); - metricsCollectionService.startAndWait(); + metricsCollectionService.startAsync().awaitRunning(); namespaceAdmin = injector.getInstance(NamespaceAdmin.class); namespaceAdmin.create(TEST_NAMESPACE_META1); namespaceAdmin.create(TEST_NAMESPACE_META2); // Restart handlers to check if they are resilient across restarts. router = injector.getInstance(NettyRouter.class); - router.startAndWait(); + router.startAsync().awaitRunning(); port = router.getBoundAddress().orElseThrow(IllegalStateException::new).getPort(); return injector; @@ -220,19 +220,25 @@ public static void stopGateway(CConfiguration conf) throws Exception { namespaceAdmin.delete(new NamespaceId(TEST_NAMESPACE1)); namespaceAdmin.delete(new NamespaceId(TEST_NAMESPACE2)); namespaceAdmin.delete(NamespaceId.DEFAULT); - appFabricServer.stopAndWait(); - appFabricProcessorService.stopAndWait(); - metricsCollectionService.stopAndWait(); - metricsQueryService.stopAndWait(); - logQueryService.stopAndWait(); - router.stopAndWait(); - datasetService.stopAndWait(); - dsOpService.stopAndWait(); - metadataService.stopAndWait(); - Closeables.closeQuietly(metadataStorage); - txService.stopAndWait(); + appFabricServer.stopAsync().awaitTerminated(); + appFabricProcessorService.stopAsync().awaitTerminated(); + metricsCollectionService.stopAsync().awaitTerminated(); + metricsQueryService.stopAsync().awaitTerminated(); + logQueryService.stopAsync().awaitTerminated(); + router.stopAsync().awaitTerminated(); + datasetService.stopAsync().awaitTerminated(); + dsOpService.stopAsync().awaitTerminated(); + metadataService.stopAsync().awaitTerminated(); + try { + if (metadataStorage != null) { + metadataStorage.close(); + } + } catch (Exception e) { + // ignore + } + txService.stopAsync().awaitTerminated(); if (messagingService instanceof Service) { - ((Service) messagingService).stopAndWait(); + ((Service) messagingService).stopAsync().awaitTerminated(); } conf.clear(); } diff --git a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/handlers/log/LogDataOffset.java b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/handlers/log/LogDataOffset.java index 5043f10847a4..e670c9238ad7 100644 --- a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/handlers/log/LogDataOffset.java +++ b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/handlers/log/LogDataOffset.java @@ -16,7 +16,7 @@ package io.cdap.cdap.gateway.handlers.log; -import com.google.common.base.Objects; + import io.cdap.cdap.logging.gateway.handlers.LogData; import io.cdap.cdap.logging.read.LogOffset; @@ -37,7 +37,7 @@ public LogData getLog() { @Override public String toString() { - return Objects.toStringHelper(this) + return com.google.common.base.MoreObjects.toStringHelper(this) .add("log", log) .add("offset", getOffset()) .toString(); diff --git a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/handlers/log/LogHttpHandlerTest.java b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/handlers/log/LogHttpHandlerTest.java index 0f9300c579c4..e338203797f5 100644 --- a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/handlers/log/LogHttpHandlerTest.java +++ b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/handlers/log/LogHttpHandlerTest.java @@ -149,16 +149,16 @@ protected void configure() { })); transactionManager = injector.getInstance(TransactionManager.class); - transactionManager.startAndWait(); + transactionManager.startAsync().awaitRunning(); StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class)); dsOpService = injector.getInstance(DatasetOpExecutorService.class); - dsOpService.startAndWait(); + dsOpService.startAsync().awaitRunning(); datasetService = injector.getInstance(DatasetService.class); - datasetService.startAndWait(); + datasetService.startAsync().awaitRunning(); logQueryService = injector.getInstance(LogQueryService.class); - logQueryService.startAndWait(); + logQueryService.startAsync().awaitRunning(); mockLogReader = (MockLogReader) injector.getInstance(LogReader.class); mockLogReader.generateLogs(); @@ -168,11 +168,11 @@ protected void configure() { @AfterClass public static void tearDown() { - logQueryService.stopAndWait(); + logQueryService.stopAsync().awaitTerminated(); - datasetService.stopAndWait(); - dsOpService.stopAndWait(); - transactionManager.stopAndWait(); + datasetService.stopAsync().awaitTerminated(); + dsOpService.stopAsync().awaitTerminated(); + transactionManager.stopAsync().awaitTerminated(); } @Test diff --git a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/handlers/log/LogLine.java b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/handlers/log/LogLine.java index 74141eebe657..f3b5d903420b 100644 --- a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/handlers/log/LogLine.java +++ b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/handlers/log/LogLine.java @@ -16,7 +16,7 @@ package io.cdap.cdap.gateway.handlers.log; -import com.google.common.base.Objects; + import io.cdap.cdap.logging.read.LogOffset; /** @@ -36,7 +36,7 @@ public String getLog() { @Override public String toString() { - return Objects.toStringHelper(this) + return com.google.common.base.MoreObjects.toStringHelper(this) .add("offset", getOffset()) .add("log", log) .toString(); diff --git a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/handlers/log/OffsetLine.java b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/handlers/log/OffsetLine.java index 2343c2d867fe..8a0b228f9b6d 100644 --- a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/handlers/log/OffsetLine.java +++ b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/handlers/log/OffsetLine.java @@ -16,7 +16,7 @@ package io.cdap.cdap.gateway.handlers.log; -import com.google.common.base.Objects; + import io.cdap.cdap.logging.read.LogOffset; /** @@ -35,7 +35,7 @@ public LogOffset getOffset() { @Override public String toString() { - return Objects.toStringHelper(this) + return com.google.common.base.MoreObjects.toStringHelper(this) .add("offset", offset) .toString(); } diff --git a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/handlers/metrics/MetricsSuiteTestBase.java b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/handlers/metrics/MetricsSuiteTestBase.java index 58375282b8b3..8916800b4e8b 100644 --- a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/handlers/metrics/MetricsSuiteTestBase.java +++ b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/handlers/metrics/MetricsSuiteTestBase.java @@ -174,20 +174,20 @@ protected void configure() { })); transactionManager = injector.getInstance(TransactionManager.class); - transactionManager.startAndWait(); + transactionManager.startAsync().awaitRunning(); StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class)); dsOpService = injector.getInstance(DatasetOpExecutorService.class); - dsOpService.startAndWait(); + dsOpService.startAsync().awaitRunning(); datasetService = injector.getInstance(DatasetService.class); - datasetService.startAndWait(); + datasetService.startAsync().awaitRunning(); metrics = injector.getInstance(MetricsQueryService.class); - metrics.startAndWait(); + metrics.startAsync().awaitRunning(); collectionService = injector.getInstance(MetricsCollectionService.class); - collectionService.startAndWait(); + collectionService.startAsync().awaitRunning(); // initialize the dataset instantiator DiscoveryServiceClient discoveryClient = injector.getInstance(DiscoveryServiceClient.class); @@ -202,11 +202,11 @@ protected void configure() { } public static void stopMetricsService(CConfiguration conf) { - collectionService.stopAndWait(); - datasetService.stopAndWait(); - dsOpService.stopAndWait(); - transactionManager.stopAndWait(); - metrics.stopAndWait(); + collectionService.stopAsync().awaitTerminated(); + datasetService.stopAsync().awaitTerminated(); + dsOpService.stopAsync().awaitTerminated(); + transactionManager.stopAsync().awaitTerminated(); + metrics.stopAsync().awaitTerminated(); conf.clear(); } diff --git a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/AuditLogTest.java b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/AuditLogTest.java index 30db4cab8093..2d5b95f4e107 100644 --- a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/AuditLogTest.java +++ b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/AuditLogTest.java @@ -103,7 +103,7 @@ public static void init() throws Exception { successValidator, new MockAccessTokenIdentityExtractor(successValidator), discoveryService, new NoOpAeadCipher()); - router.startAndWait(); + router.startAsync().awaitRunning(); httpService = NettyHttpService.builder("test").setHttpHandlers(new TestHandler()).build(); httpService.start(); @@ -119,7 +119,7 @@ public static void init() throws Exception { public static void finish() throws Exception { cancelDiscovery.cancel(); httpService.stop(); - router.stopAndWait(); + router.stopAsync().awaitTerminated(); } @Test diff --git a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/AuthServerAnnounceTest.java b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/AuthServerAnnounceTest.java index 2bffff417718..010a18914466 100644 --- a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/AuthServerAnnounceTest.java +++ b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/AuthServerAnnounceTest.java @@ -145,12 +145,12 @@ protected void startUp() { new RouterServiceLookup(cConf, (DiscoveryServiceClient) discoveryService, new RouterPathLookup()), validator, userIdentityExtractor, discoveryServiceClient, new NoOpAeadCipher()); - router.startAndWait(); + router.startAsync().awaitRunning(); } @Override protected void shutDown() { - router.stopAndWait(); + router.stopAsync().awaitTerminated(); } InetSocketAddress getRouterAddress() { diff --git a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/ConfigBasedRequestBlockingTest.java b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/ConfigBasedRequestBlockingTest.java index 48ff938ef4bf..8b5f23c813bf 100644 --- a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/ConfigBasedRequestBlockingTest.java +++ b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/ConfigBasedRequestBlockingTest.java @@ -68,7 +68,7 @@ public static void init() throws Exception { successValidator, new MockAccessTokenIdentityExtractor(successValidator), discoveryService, new NoOpAeadCipher()); - router.startAndWait(); + router.startAsync().awaitRunning(); httpService = NettyHttpService.builder("test").setHttpHandlers(new AuditLogTest.TestHandler()) .build(); @@ -121,7 +121,7 @@ public void testRouterStatus() throws Exception { public static void finish() throws Exception { cancelDiscovery.cancel(); httpService.stop(); - router.stopAndWait(); + router.stopAsync().awaitTerminated(); } private void testGet(int expectedStatus, String expectedResponse, String path) diff --git a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterHttpTest.java b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterHttpTest.java index 3169cc9caa25..3ae37f5f1597 100644 --- a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterHttpTest.java +++ b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterHttpTest.java @@ -107,12 +107,12 @@ protected void startUp() { new RouterPathLookup()), new SuccessTokenValidator(), userIdentityExtractor, discoveryServiceClient, new NoOpAeadCipher()); - router.startAndWait(); + router.startAsync().awaitRunning(); } @Override protected void shutDown() { - router.stopAndWait(); + router.stopAsync().awaitTerminated(); } public InetSocketAddress getRouterAddress() { diff --git a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterHttpsTest.java b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterHttpsTest.java index e112b344a1a7..6468bea92943 100644 --- a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterHttpsTest.java +++ b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterHttpsTest.java @@ -201,12 +201,12 @@ protected void startUp() { new RouterPathLookup()), new SuccessTokenValidator(), userIdentityExtractor, discoveryServiceClient, new NoOpAeadCipher()); - router.startAndWait(); + router.startAsync().awaitRunning(); } @Override protected void shutDown() { - router.stopAndWait(); + router.stopAsync().awaitTerminated(); } public InetSocketAddress getRouterAddress() { diff --git a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterPipelineTest.java b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterPipelineTest.java index 422557635380..3adc2b6e40e8 100644 --- a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterPipelineTest.java +++ b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterPipelineTest.java @@ -209,7 +209,9 @@ private void deploy(int num) throws Exception { LocationFactory lf = new LocalLocationFactory(TMP_FOLDER.newFolder()); Location programJar = AppJarHelper.createDeploymentJar(lf, DummyApp.class); - GATEWAY_SERVER.setExpectedJarBytes(ByteStreams.toByteArray(Locations.newInputSupplier(programJar))); + try (java.io.InputStream is = programJar.getInputStream()) { + GATEWAY_SERVER.setExpectedJarBytes(ByteStreams.toByteArray(is)); + } for (int i = 0; i < num; i++) { LOG.info("Deploying {}/{}", i, num); @@ -220,7 +222,9 @@ private void deploy(int num) throws Exception { urlConn.setDoOutput(true); urlConn.setDoInput(true); - ByteStreams.copy(Locations.newInputSupplier(programJar), urlConn.getOutputStream()); + try (java.io.InputStream is = programJar.getInputStream()) { + ByteStreams.copy(is, urlConn.getOutputStream()); + } Assert.assertEquals(200, urlConn.getResponseCode()); urlConn.getInputStream().close(); urlConn.disconnect(); diff --git a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterTestBase.java b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterTestBase.java index 06ec876f3b8e..1808fd0a66be 100644 --- a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterTestBase.java +++ b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterTestBase.java @@ -148,12 +148,10 @@ private String resolveUri(String path) throws URISyntaxException { @Before public void startUp() throws Exception { routerService = createRouterService(HOSTNAME, discoveryService); - List> futures = new ArrayList<>(); - futures.add(routerService.start()); + routerService.startAsync().awaitRunning(); for (ServerService server : allServers) { - futures.add(server.start()); + server.startAsync().awaitRunning(); } - Futures.allAsList(futures).get(); // Wait for both servers of defaultService to be registered ServiceDiscovered discover = ((DiscoveryServiceClient) discoveryService) @@ -173,12 +171,10 @@ public void onChange(ServiceDiscovered serviceDiscovered) { @After public void tearDown() throws Exception { - List> futures = new ArrayList<>(); for (ServerService server : allServers) { - futures.add(server.stop()); + server.stopAsync().awaitTerminated(); } - futures.add(routerService.stop()); - Futures.successfulAsList(futures).get(); + routerService.stopAsync().awaitTerminated(); } @Test @@ -531,7 +527,7 @@ public void testConnectionClose2() throws Exception { }); t.start(); - defaultServer1.stopAndWait(); + defaultServer1.stopAsync().awaitTerminated(); Assert.assertEquals(200, result.get().intValue()); Assert.assertEquals(1, defaultServer1.getNumRequests()); Assert.assertEquals(1, defaultServer2.getNumRequests()); @@ -561,7 +557,7 @@ public void testConfigReloading() throws Exception { successValidator, new MockAccessTokenIdentityExtractor(successValidator), discoveryService, new NoOpAeadCipher()); - router1.startAndWait(); + router1.startAsync().awaitRunning(); // Configure router with config-reloading time set to 0 CConfiguration cConfSpy2 = Mockito.spy(CConfiguration.create()); @@ -573,15 +569,15 @@ public void testConfigReloading() throws Exception { successValidator, new MockAccessTokenIdentityExtractor(successValidator), discoveryService, new NoOpAeadCipher()); - router2.startAndWait(); + router2.startAsync().awaitRunning(); // Wait sometime for cConf to reload Thread.sleep(TimeUnit.MILLISECONDS.convert(reloadIntervalSeconds + 2, TimeUnit.SECONDS)); Mockito.verify(cConfSpy1, Mockito.times(1)).reloadConfiguration(); Mockito.verify(cConfSpy2, Mockito.never()).reloadConfiguration(); - router1.stopAndWait(); - router2.stopAndWait(); + router1.stopAsync().awaitTerminated(); + router2.stopAsync().awaitTerminated(); } protected HttpURLConnection openUrl(URL url) throws Exception { diff --git a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/RouterResource.java b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/RouterResource.java index cca9d8d4bfc7..e633516c33f6 100644 --- a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/RouterResource.java +++ b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/RouterResource.java @@ -77,12 +77,12 @@ protected void before() { new RouterServiceLookup(cConf, (DiscoveryServiceClient) discoveryService, new RouterPathLookup()), mockValidator, extractor, discoveryServiceClient, new NoOpAeadCipher()); - router.startAndWait(); + router.startAsync().awaitRunning(); } @Override protected void after() { - router.stopAndWait(); + router.stopAsync().awaitTerminated(); } InetSocketAddress getRouterAddress() { diff --git a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/RoutingToDataSetsTest.java b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/RoutingToDataSetsTest.java index b565b5ce1067..d1ef6f2d3fb4 100644 --- a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/RoutingToDataSetsTest.java +++ b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/RoutingToDataSetsTest.java @@ -84,21 +84,21 @@ public static void before() throws Exception { new RouterServiceLookup(cConf, discoveryServiceClient, new RouterPathLookup()), new SuccessTokenValidator(), userIdentityExtractor, discoveryServiceClient, new NoOpAeadCipher()); - nettyRouter.startAndWait(); + nettyRouter.startAsync().awaitRunning(); // Starting mock DataSet service DiscoveryService discoveryService = injector.getInstance(DiscoveryService.class); mockService = new MockHttpService(discoveryService, Constants.Service.DATASET_MANAGER, new MockDatasetTypeHandler(), new MockDatasetInstanceHandler()); - mockService.startAndWait(); + mockService.startAsync().awaitRunning(); } @AfterClass public static void after() { try { - nettyRouter.stopAndWait(); + nettyRouter.stopAsync().awaitTerminated(); } finally { - mockService.stopAndWait(); + mockService.stopAsync().awaitTerminated(); } } diff --git a/cdap-integration-test/pom.xml b/cdap-integration-test/pom.xml index 95a96dc6114d..0e6387163377 100644 --- a/cdap-integration-test/pom.xml +++ b/cdap-integration-test/pom.xml @@ -106,4 +106,18 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + diff --git a/cdap-kafka/pom.xml b/cdap-kafka/pom.xml index dfd2533e6d5b..8f1fe9b040ab 100644 --- a/cdap-kafka/pom.xml +++ b/cdap-kafka/pom.xml @@ -46,6 +46,20 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + dist @@ -57,7 +71,6 @@ org.apache.maven.plugins maven-jar-plugin - 2.4 org.apache.maven.plugins diff --git a/cdap-kafka/src/main/java/io/cdap/cdap/kafka/run/KafkaServerMain.java b/cdap-kafka/src/main/java/io/cdap/cdap/kafka/run/KafkaServerMain.java index ccf6c3bd4174..ca316ea394ab 100644 --- a/cdap-kafka/src/main/java/io/cdap/cdap/kafka/run/KafkaServerMain.java +++ b/cdap-kafka/src/main/java/io/cdap/cdap/kafka/run/KafkaServerMain.java @@ -78,12 +78,12 @@ public void init(String[] args) { client.create(path, null, CreateMode.PERSISTENT), KeeperException.NodeExistsException.class, path).get(); - client.stopAndWait(); + client.stopAsync().awaitTerminated(); zkConnectStr = String.format("%s/%s", zkConnectStr, zkNamespace); } catch (Exception e) { throw Throwables.propagate(e); } finally { - client.stopAndWait(); + client.stopAsync().awaitTerminated(); } } @@ -122,11 +122,7 @@ public void start() { LOG.info("Starting embedded kafka server..."); kafkaServer = new EmbeddedKafkaServer(kafkaProperties); - Service.State state = kafkaServer.startAndWait(); - - if (state != Service.State.RUNNING) { - throw new IllegalStateException("Kafka server has not started... terminating."); - } + kafkaServer.startAsync().awaitRunning(); LOG.info("Embedded kafka server started successfully."); } @@ -135,7 +131,7 @@ public void start() { public void stop() { LOG.info("Stopping embedded kafka server..."); if (kafkaServer != null && kafkaServer.isRunning()) { - kafkaServer.stopAndWait(); + kafkaServer.stopAsync().awaitTerminated(); } } diff --git a/cdap-kms/pom.xml b/cdap-kms/pom.xml index 2ce525c0dd5a..b709c14a601f 100644 --- a/cdap-kms/pom.xml +++ b/cdap-kms/pom.xml @@ -30,6 +30,17 @@ jar io.cdap.cdap.cdap-kms-${project.version} + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + diff --git a/cdap-kubernetes/pom.xml b/cdap-kubernetes/pom.xml index 11cf45e0eaa4..c56077bb8cf9 100644 --- a/cdap-kubernetes/pom.xml +++ b/cdap-kubernetes/pom.xml @@ -151,6 +151,17 @@ ${testSourceLocation} + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + @@ -184,7 +195,6 @@ org.apache.maven.plugins maven-jar-plugin - 2.4 jar @@ -192,6 +202,7 @@ ${project.build.directory}/libexec ${project.groupId}.${project.build.finalName} + false jar diff --git a/cdap-log-publisher-spi/pom.xml b/cdap-log-publisher-spi/pom.xml index 0fd212569a81..69304e24bd07 100644 --- a/cdap-log-publisher-spi/pom.xml +++ b/cdap-log-publisher-spi/pom.xml @@ -34,4 +34,18 @@ ch.qos.logback + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + diff --git a/cdap-master-spi/pom.xml b/cdap-master-spi/pom.xml index bc76a8bee69d..50106f9fc82b 100644 --- a/cdap-master-spi/pom.xml +++ b/cdap-master-spi/pom.xml @@ -46,4 +46,18 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + diff --git a/cdap-master/pom.xml b/cdap-master/pom.xml index 77a034831d25..fc6166c7a24a 100644 --- a/cdap-master/pom.xml +++ b/cdap-master/pom.xml @@ -373,7 +373,6 @@ org.apache.maven.plugins maven-jar-plugin - 2.4 org.apache.maven.plugins @@ -1117,6 +1116,15 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 17 + 17 + + org.apache.maven.plugins maven-surefire-plugin diff --git a/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MasterServiceMain.java b/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MasterServiceMain.java index c2bc17f49069..6b59970033ba 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MasterServiceMain.java +++ b/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MasterServiceMain.java @@ -234,7 +234,7 @@ public MasterServiceMain() { this.leaderElection = new LeaderElection(zkClient, electionPath, electionHandler); // leader election will normally stay running. Will only stop if there was some issue starting up. - this.leaderElection.addListener(new ServiceListenerAdapter() { + this.leaderElection.addListener(new Service.Listener() { @Override public void terminated(Service.State from) { if (!stopped) { @@ -250,7 +250,7 @@ public void failed(Service.State from, Throwable failure) { System.exit(1); } } - }, MoreExecutors.sameThreadExecutor()); + }, MoreExecutors.directExecutor()); } @Override @@ -273,8 +273,8 @@ public void start() throws Exception { // Tries to create the ZK root node (which can be namespaced through the zk connection string) Futures.getUnchecked(ZKOperations.ignoreError(zkClient.create("/", null, CreateMode.PERSISTENT), KeeperException.NodeExistsException.class, null)); - electionInfoService.startAndWait(); - leaderElection.startAndWait(); + electionInfoService.startAsync().awaitRunning(); + leaderElection.startAsync().awaitRunning(); } @Override @@ -333,7 +333,13 @@ public void stop() { } stopQuietly(electionInfoService); stopQuietly(zkClient); - Closeables.closeQuietly(logAppenderInitializer); + if (logAppenderInitializer instanceof AutoCloseable) { + try { + ((AutoCloseable) logAppenderInitializer).close(); + } catch (Exception e) { + // Ignore + } + } } @Override @@ -400,7 +406,7 @@ private boolean checkDirectoryExists(FileContext fileContext, org.apache.hadoop. private static T getAndStart(Injector injector, Class cls) { T service = injector.getInstance(cls); LOG.debug("Starting service in master {}", service); - service.startAndWait(); + service.startAsync().awaitRunning(); LOG.info("Service {} started in master", service); return service; } @@ -412,7 +418,7 @@ private static void stopQuietly(@Nullable Service service) { try { if (service != null) { LOG.debug("Stopping service in master: {}", service); - service.stopAndWait(); + service.stopAsync().awaitTerminated(); LOG.info("Service {} stopped in master", service); } } catch (Exception e) { @@ -672,7 +678,7 @@ public void leader() { } LOG.info("Starting service in master: {}", service); try { - service.startAndWait(); + service.startAsync().awaitRunning(); } catch (Throwable t) { // shut down the executor and stop the twill app, // then throw an exception to cause the leader election service to stop @@ -714,9 +720,27 @@ private void stop(boolean stopRequested) { stopQuietly(service); } services.clear(); - Closeables.closeQuietly(metadataStorage); - Closeables.closeQuietly(accessControllerInstantiator); - Closeables.closeQuietly(logAppenderInitializer); + if (metadataStorage instanceof AutoCloseable) { + try { + ((AutoCloseable) metadataStorage).close(); + } catch (Exception e) { + // Ignore + } + } + if (accessControllerInstantiator instanceof AutoCloseable) { + try { + ((AutoCloseable) accessControllerInstantiator).close(); + } catch (Exception e) { + // Ignore + } + } + if (logAppenderInitializer instanceof AutoCloseable) { + try { + ((AutoCloseable) logAppenderInitializer).close(); + } catch (Exception e) { + // Ignore + } + } } /** diff --git a/cdap-master/src/main/java/io/cdap/cdap/data/tools/JobQueueDebugger.java b/cdap-master/src/main/java/io/cdap/cdap/data/tools/JobQueueDebugger.java index fc5d98e3df13..c1d1884f36cc 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/data/tools/JobQueueDebugger.java +++ b/cdap-master/src/main/java/io/cdap/cdap/data/tools/JobQueueDebugger.java @@ -125,12 +125,12 @@ public JobQueueDebugger(CConfiguration cConf, ZKClientService zkClientService, @Override protected void startUp() { - zkClientService.startAndWait(); + zkClientService.startAsync().awaitRunning(); } @Override protected void shutDown() { - zkClientService.stopAndWait(); + zkClientService.stopAsync().awaitTerminated(); } private JobQueueScanner getJobQueueScanner() { @@ -221,8 +221,8 @@ private JobStatistics scanPartition(final int partition, boolean trace) { private boolean scanJobQueue(JobQueue jobQueue, int partition, JobStatistics jobStatistics) throws IOException { try (CloseableIterator jobs = jobQueue.getJobs(partition, lastJobConsumed)) { - Stopwatch stopwatch = new Stopwatch().start(); - while (stopwatch.elapsedMillis() < 1000) { + Stopwatch stopwatch = Stopwatch.createStarted(); + while (stopwatch.elapsed(java.util.concurrent.TimeUnit.MILLISECONDS) < 1000) { if (!jobs.hasNext()) { lastJobConsumed = null; return false; @@ -422,7 +422,7 @@ public static void main(String[] args) throws Exception { } JobQueueDebugger debugger = createDebugger(); - debugger.startAndWait(); + debugger.startAsync().awaitRunning(); debugger.printTopicMessageIds(); @@ -431,6 +431,6 @@ public static void main(String[] args) throws Exception { } else { debugger.scanPartition(partition, trace); } - debugger.stopAndWait(); + debugger.stopAsync().awaitTerminated(); } } diff --git a/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/MetadataServiceMain.java b/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/MetadataServiceMain.java index 8ef580633609..ca017464c34f 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/MetadataServiceMain.java +++ b/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/MetadataServiceMain.java @@ -145,7 +145,13 @@ protected void doStart() { @Override protected void doStop() { - Closeables.closeQuietly(metadataStorage); + if (metadataStorage instanceof AutoCloseable) { + try { + ((AutoCloseable) metadataStorage).close(); + } catch (Exception e) { + // Ignore + } + } notifyStopped(); } }); diff --git a/cdap-master/src/test/java/io/cdap/cdap/master/environment/MockMasterEnvironment.java b/cdap-master/src/test/java/io/cdap/cdap/master/environment/MockMasterEnvironment.java index 534f9926257a..4651aed50343 100644 --- a/cdap-master/src/test/java/io/cdap/cdap/master/environment/MockMasterEnvironment.java +++ b/cdap-master/src/test/java/io/cdap/cdap/master/environment/MockMasterEnvironment.java @@ -42,7 +42,7 @@ public class MockMasterEnvironment implements MasterEnvironment { @Override public void initialize(MasterEnvironmentContext context) { zkClient = ZKClientService.Builder.of(context.getConfigurations().get(Constants.Zookeeper.QUORUM)).build(); - zkClient.startAndWait(); + zkClient.startAsync().awaitRunning(); discoveryService = new ZKDiscoveryService(zkClient); twillRunnerService = new NoopTwillRunnerService(); @@ -56,7 +56,7 @@ public void initialize(MasterEnvironmentContext context) { @Override public void destroy() { discoveryService.close(); - zkClient.stopAndWait(); + zkClient.stopAsync().awaitTerminated(); } @Override diff --git a/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/LogsServiceMainTest.java b/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/LogsServiceMainTest.java index dfea32098ff6..1100ca241b62 100644 --- a/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/LogsServiceMainTest.java +++ b/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/LogsServiceMainTest.java @@ -20,7 +20,7 @@ import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.classic.spi.LoggingEvent; import ch.qos.logback.core.AppenderBase; -import com.google.common.base.Objects; + import com.google.common.collect.ImmutableList; import com.google.common.net.HttpHeaders; import com.google.gson.Gson; @@ -168,7 +168,7 @@ LogData getLog() { @Override public String toString() { - return Objects.toStringHelper(this) + return com.google.common.base.MoreObjects.toStringHelper(this) .add("log", log) .add("offset", getOffset()) .toString(); @@ -188,7 +188,7 @@ LogOffset getOffset() { @Override public String toString() { - return Objects.toStringHelper(this) + return com.google.common.base.MoreObjects.toStringHelper(this) .add("offset", offset) .toString(); } diff --git a/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/PreviewServiceMainTest.java b/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/PreviewServiceMainTest.java index 1b9a0287a086..48fdcc40f924 100644 --- a/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/PreviewServiceMainTest.java +++ b/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/PreviewServiceMainTest.java @@ -99,7 +99,7 @@ public static void initPreviewService() throws Exception { cConf.set(Constants.CFG_LOCAL_DATA_DIR, temporaryFolder.newFolder().getAbsolutePath()); Injector injector = ArtifactLocalizerTwillRunnable.createInjector(cConf, new Configuration()); artifactLocalizerService = injector.getInstance(ArtifactLocalizerService.class); - artifactLocalizerService.startAndWait(); + artifactLocalizerService.startAsync().awaitRunning(); // Start the preview service main, which will use its own local datadir and fetch artifacts from app-fabric via // the artifact localizer service. startService(PreviewServiceMain.class); @@ -108,7 +108,7 @@ public static void initPreviewService() throws Exception { @AfterClass public static void afterPreviewService() throws Exception { stopService(PreviewServiceMain.class); - artifactLocalizerService.stopAndWait(); + artifactLocalizerService.stopAsync().awaitTerminated(); } @Test diff --git a/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/SystemMetricsExporterServiceMainTest.java b/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/SystemMetricsExporterServiceMainTest.java index 3ce04353d3a4..eb6ce216f9e9 100644 --- a/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/SystemMetricsExporterServiceMainTest.java +++ b/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/SystemMetricsExporterServiceMainTest.java @@ -38,9 +38,9 @@ public void testSystemMetricsExporterService() { Map metricTags = ImmutableMap.of("key1", "value1", "key2", "value2"); JmxMetricsCollector metricsCollector = factory.create(metricTags); // JMX server isn't running, but that shouldn't raise exceptions, errors will be logged. - metricsCollector.startAndWait(); + metricsCollector.startAsync().awaitRunning(); Assert.assertTrue(metricsCollector.isRunning()); - metricsCollector.stopAndWait(); + metricsCollector.stopAsync().awaitTerminated(); Assert.assertFalse(metricsCollector.isRunning()); } diff --git a/cdap-master/src/test/java/io/cdap/cdap/metrics/jmx/JmxMetricsCollectorTest.java b/cdap-master/src/test/java/io/cdap/cdap/metrics/jmx/JmxMetricsCollectorTest.java index af9495a68d15..923ca5c7993f 100644 --- a/cdap-master/src/test/java/io/cdap/cdap/metrics/jmx/JmxMetricsCollectorTest.java +++ b/cdap-master/src/test/java/io/cdap/cdap/metrics/jmx/JmxMetricsCollectorTest.java @@ -98,7 +98,7 @@ public void testNumberOfMetricsEmitted() throws InterruptedException, MalformedU cConf.setInt(Constants.JmxMetricsCollector.POLL_INTERVAL_SECS, 1); Map metricTags = ImmutableMap.of("key1", "value1", "key2", "value2"); JmxMetricsCollector jmxMetrics = new JmxMetricsCollector(cConf, publisher, metricTags); - jmxMetrics.startAndWait(); + jmxMetrics.startAsync().awaitRunning(); verify(publisher, times(1)).initialize(); // Poll should run at 0, 1. 2 secs buffer. Tasks.waitFor(true, () -> { @@ -109,6 +109,6 @@ public void testNumberOfMetricsEmitted() throws InterruptedException, MalformedU } return true; }, 3, TimeUnit.SECONDS); - jmxMetrics.stop(); + jmxMetrics.stopAsync().awaitTerminated(); } } diff --git a/cdap-messaging-ext-spanner/pom.xml b/cdap-messaging-ext-spanner/pom.xml index 0ab710afeece..23984afe8364 100644 --- a/cdap-messaging-ext-spanner/pom.xml +++ b/cdap-messaging-ext-spanner/pom.xml @@ -65,6 +65,20 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + dist @@ -96,7 +110,6 @@ org.apache.maven.plugins maven-jar-plugin - 2.4 jar @@ -104,6 +117,7 @@ ${project.build.directory}/libexec ${project.groupId}.${project.build.finalName} + false jar diff --git a/cdap-messaging-spi/pom.xml b/cdap-messaging-spi/pom.xml index de1d6b4c0dad..29c959074c2b 100644 --- a/cdap-messaging-spi/pom.xml +++ b/cdap-messaging-spi/pom.xml @@ -51,4 +51,18 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + \ No newline at end of file diff --git a/cdap-metadata-ext-spanner/pom.xml b/cdap-metadata-ext-spanner/pom.xml index 06af96381b57..a70c1375693a 100644 --- a/cdap-metadata-ext-spanner/pom.xml +++ b/cdap-metadata-ext-spanner/pom.xml @@ -69,6 +69,20 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + @@ -103,6 +117,7 @@ ${project.groupId}.${project.build.finalName} ${project.build.directory}/libexec + false jar @@ -112,7 +127,6 @@ org.apache.maven.plugins - 2.4 diff --git a/cdap-metadata-spi/pom.xml b/cdap-metadata-spi/pom.xml index 6e3cacfaadc2..44861c9c1402 100644 --- a/cdap-metadata-spi/pom.xml +++ b/cdap-metadata-spi/pom.xml @@ -63,10 +63,18 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + org.apache.maven.plugins maven-jar-plugin - 2.4 test-jar diff --git a/cdap-operational-stats-core/pom.xml b/cdap-operational-stats-core/pom.xml index e8e771edf9cd..5b914acea424 100644 --- a/cdap-operational-stats-core/pom.xml +++ b/cdap-operational-stats-core/pom.xml @@ -73,6 +73,20 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + dist @@ -81,7 +95,6 @@ org.apache.maven.plugins maven-jar-plugin - 2.4 jar @@ -89,6 +102,7 @@ ${project.build.directory} ${project.groupId}.${project.build.finalName} + false jar @@ -100,6 +114,7 @@ ${project.build.directory} ${project.groupId}.${project.build.finalName}.sdk + false jar diff --git a/cdap-proto/pom.xml b/cdap-proto/pom.xml index 56289c2b8916..4586e71985a6 100644 --- a/cdap-proto/pom.xml +++ b/cdap-proto/pom.xml @@ -53,4 +53,18 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 17 + 17 + + + + + diff --git a/cdap-runtime-ext-dataproc/pom.xml b/cdap-runtime-ext-dataproc/pom.xml index 1a215707834a..03c33900fe4d 100644 --- a/cdap-runtime-ext-dataproc/pom.xml +++ b/cdap-runtime-ext-dataproc/pom.xml @@ -149,6 +149,20 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + @@ -235,7 +249,6 @@ org.apache.maven.plugins maven-jar-plugin - 2.4 jar @@ -243,6 +256,7 @@ ${project.build.directory}/libexec ${project.groupId}.${project.build.finalName} + false jar diff --git a/cdap-runtime-ext-dataproc/src/main/java/io/cdap/cdap/runtime/spi/runtimejob/DataprocRuntimeEnvironment.java b/cdap-runtime-ext-dataproc/src/main/java/io/cdap/cdap/runtime/spi/runtimejob/DataprocRuntimeEnvironment.java index 85fdaf7851a5..6629fd200099 100644 --- a/cdap-runtime-ext-dataproc/src/main/java/io/cdap/cdap/runtime/spi/runtimejob/DataprocRuntimeEnvironment.java +++ b/cdap-runtime-ext-dataproc/src/main/java/io/cdap/cdap/runtime/spi/runtimejob/DataprocRuntimeEnvironment.java @@ -70,7 +70,7 @@ public void initialize(String sparkCompat, String launchMode) throws Exception { addConsoleAppender(); System.setProperty(TWILL_ZK_SERVER_LOCALHOST, "false"); zkServer = InMemoryZKServer.builder().build(); - zkServer.startAndWait(); + zkServer.startAsync().awaitRunning(); InetSocketAddress resolved = resolve(zkServer.getLocalAddress()); String connectionStr = resolved.getHostString() + ":" + resolved.getPort(); @@ -111,7 +111,7 @@ public void destroy() { yarnTwillRunnerService.stop(); } if (zkServer != null) { - zkServer.stopAndWait(); + zkServer.stopAsync().awaitTerminated(); } if (locationFactory != null) { Location location = locationFactory.create("/"); diff --git a/cdap-runtime-ext-emr/pom.xml b/cdap-runtime-ext-emr/pom.xml index 399dcbca846b..fd83ee5326e1 100644 --- a/cdap-runtime-ext-emr/pom.xml +++ b/cdap-runtime-ext-emr/pom.xml @@ -95,6 +95,20 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + dist @@ -126,7 +140,6 @@ org.apache.maven.plugins maven-jar-plugin - 2.4 jar @@ -134,6 +147,7 @@ ${project.build.directory}/libexec ${project.groupId}.${project.build.finalName} + false jar diff --git a/cdap-runtime-ext-remote-hadoop/pom.xml b/cdap-runtime-ext-remote-hadoop/pom.xml index 31c288a14624..7fb035de61ce 100644 --- a/cdap-runtime-ext-remote-hadoop/pom.xml +++ b/cdap-runtime-ext-remote-hadoop/pom.xml @@ -49,6 +49,20 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + dist @@ -80,7 +94,6 @@ org.apache.maven.plugins maven-jar-plugin - 2.4 jar @@ -88,6 +101,7 @@ ${project.build.directory}/libexec ${project.groupId}.${project.build.finalName} + false jar diff --git a/cdap-runtime-spi/pom.xml b/cdap-runtime-spi/pom.xml index adf76d6bb96e..08e48c1a82c7 100644 --- a/cdap-runtime-spi/pom.xml +++ b/cdap-runtime-spi/pom.xml @@ -51,4 +51,18 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + diff --git a/cdap-securestore-ext-cloudkms/pom.xml b/cdap-securestore-ext-cloudkms/pom.xml index c504c1c43c46..aa912ad6847f 100644 --- a/cdap-securestore-ext-cloudkms/pom.xml +++ b/cdap-securestore-ext-cloudkms/pom.xml @@ -83,6 +83,20 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + dist @@ -114,7 +128,6 @@ org.apache.maven.plugins maven-jar-plugin - 2.4 jar @@ -122,6 +135,7 @@ ${project.build.directory}/libexec ${project.groupId}.${project.build.finalName} + false jar diff --git a/cdap-securestore-ext-gcp-secretstore/pom.xml b/cdap-securestore-ext-gcp-secretstore/pom.xml index 4a138a5f63cd..faeda1f7bb64 100644 --- a/cdap-securestore-ext-gcp-secretstore/pom.xml +++ b/cdap-securestore-ext-gcp-secretstore/pom.xml @@ -75,6 +75,20 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + dist @@ -106,7 +120,6 @@ org.apache.maven.plugins maven-jar-plugin - 2.4 jar @@ -114,6 +127,7 @@ ${project.build.directory}/libexec ${project.groupId}.${project.build.finalName} + false jar diff --git a/cdap-securestore-spi/pom.xml b/cdap-securestore-spi/pom.xml index 7d5a730ff16e..a0934efe4ee9 100644 --- a/cdap-securestore-spi/pom.xml +++ b/cdap-securestore-spi/pom.xml @@ -40,4 +40,18 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + diff --git a/cdap-security-spi/pom.xml b/cdap-security-spi/pom.xml index 65c1f767aaaf..04e9026ac7dd 100644 --- a/cdap-security-spi/pom.xml +++ b/cdap-security-spi/pom.xml @@ -49,10 +49,18 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + org.apache.maven.plugins maven-jar-plugin - 2.4 test-jar diff --git a/cdap-security/pom.xml b/cdap-security/pom.xml index b534ac3ee133..4bca22559770 100644 --- a/cdap-security/pom.xml +++ b/cdap-security/pom.xml @@ -187,10 +187,18 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 17 + 17 + + org.apache.maven.plugins maven-jar-plugin - 2.4 test-jar @@ -214,7 +222,6 @@ org.apache.maven.plugins maven-jar-plugin - 2.4 org.apache.maven.plugins diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/auth/AccessToken.java b/cdap-security/src/main/java/io/cdap/cdap/security/auth/AccessToken.java index 9d2941fc1479..52df922d4448 100644 --- a/cdap-security/src/main/java/io/cdap/cdap/security/auth/AccessToken.java +++ b/cdap-security/src/main/java/io/cdap/cdap/security/auth/AccessToken.java @@ -16,7 +16,7 @@ package io.cdap.cdap.security.auth; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.collect.Maps; import io.cdap.cdap.api.common.Bytes; import io.cdap.cdap.api.data.schema.Schema; @@ -110,7 +110,7 @@ public byte[] getDigestBytes() { public boolean equals(Object object) { if (object instanceof AccessToken) { AccessToken other = (AccessToken) object; - return Objects.equal(identifier, other.identifier) + return java.util.Objects.equals(identifier, other.identifier) && keyId == other.keyId && Bytes.equals(digest, other.digest); } @@ -119,12 +119,12 @@ public boolean equals(Object object) { @Override public int hashCode() { - return Objects.hashCode(getIdentifier(), getKeyId(), Bytes.hashCode(getDigestBytes())); + return java.util.Objects.hash(getIdentifier(), getKeyId(), Bytes.hashCode(getDigestBytes())); } @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("identifier", identifier) .add("keyId", keyId) .add("digest", Bytes.toStringBinary(digest)) diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/auth/AccessTokenValidator.java b/cdap-security/src/main/java/io/cdap/cdap/security/auth/AccessTokenValidator.java index 25a4896eed36..3a47f348b64d 100644 --- a/cdap-security/src/main/java/io/cdap/cdap/security/auth/AccessTokenValidator.java +++ b/cdap-security/src/main/java/io/cdap/cdap/security/auth/AccessTokenValidator.java @@ -42,13 +42,13 @@ public AccessTokenValidator(TokenManager tokenManager, Codec access @Override protected void startUp() throws Exception { LOG.info("Starting up AccessTokenValidator service"); - tokenManager.startAndWait(); + tokenManager.startAsync().awaitRunning(); } @Override protected void shutDown() throws Exception { LOG.info("Shutting down AccessTokenValidator service"); - tokenManager.stopAndWait(); + tokenManager.stopAsync().awaitTerminated(); } @Override diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/auth/DistributedKeyManager.java b/cdap-security/src/main/java/io/cdap/cdap/security/auth/DistributedKeyManager.java index 2b0493c7b196..fd3ea398c8aa 100644 --- a/cdap-security/src/main/java/io/cdap/cdap/security/auth/DistributedKeyManager.java +++ b/cdap-security/src/main/java/io/cdap/cdap/security/auth/DistributedKeyManager.java @@ -114,7 +114,7 @@ public void follower() { LOG.debug("Transitioned to follower"); } }); - this.leaderElection.start(); + this.leaderElection.startAsync(); startExpirationThread(); } @@ -123,7 +123,7 @@ public void shutDown() { if (timer != null) { timer.cancel(); } - leaderElection.stopAndWait(); + leaderElection.stopAsync().awaitTerminated(); } @Override diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/auth/KeyIdentifier.java b/cdap-security/src/main/java/io/cdap/cdap/security/auth/KeyIdentifier.java index 69f5d2a81bb5..0c1ce82546a8 100644 --- a/cdap-security/src/main/java/io/cdap/cdap/security/auth/KeyIdentifier.java +++ b/cdap-security/src/main/java/io/cdap/cdap/security/auth/KeyIdentifier.java @@ -16,7 +16,7 @@ package io.cdap.cdap.security.auth; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.collect.Maps; import io.cdap.cdap.api.data.schema.Schema; import java.util.Arrays; @@ -93,20 +93,20 @@ public boolean equals(Object object) { KeyIdentifier other = (KeyIdentifier) object; return Arrays.equals(encodedKey, other.encodedKey) && keyId == other.keyId && algorithm.equals(other.algorithm) - && Objects.equal(getKey(), other.getKey()) - && Objects.equal(expiration, other.expiration); + && java.util.Objects.equals(getKey(), other.getKey()) + && java.util.Objects.equals(expiration, other.expiration); } return false; } @Override public int hashCode() { - return Objects.hashCode(getKey(), getKeyId(), encodedKey, algorithm, expiration); + return java.util.Objects.hash(getKey(), getKeyId(), encodedKey, algorithm, expiration); } @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("keyId", keyId) .add("expiration", expiration) .toString(); diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/auth/TokenManager.java b/cdap-security/src/main/java/io/cdap/cdap/security/auth/TokenManager.java index 3749baf1e6fb..99fe95b6d723 100644 --- a/cdap-security/src/main/java/io/cdap/cdap/security/auth/TokenManager.java +++ b/cdap-security/src/main/java/io/cdap/cdap/security/auth/TokenManager.java @@ -44,13 +44,13 @@ public TokenManager(KeyManager keyManager, Codec identifierCodec) @Override public void startUp() { LOG.info("Starting TokenManager service"); - this.keyManager.startAndWait(); + this.keyManager.startAsync().awaitRunning(); } @Override public void shutDown() { LOG.info("Shutting down TokenManager service."); - this.keyManager.stopAndWait(); + this.keyManager.stopAsync().awaitTerminated(); } /** diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/auth/UserIdentity.java b/cdap-security/src/main/java/io/cdap/cdap/security/auth/UserIdentity.java index c1690d982a77..314db7faf136 100644 --- a/cdap-security/src/main/java/io/cdap/cdap/security/auth/UserIdentity.java +++ b/cdap-security/src/main/java/io/cdap/cdap/security/auth/UserIdentity.java @@ -16,7 +16,7 @@ package io.cdap.cdap.security.auth; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import io.cdap.cdap.api.data.schema.Schema; @@ -146,15 +146,15 @@ public boolean equals(Object other) { } UserIdentity otherToken = (UserIdentity) other; - return Objects.equal(username, otherToken.username) - && Objects.equal(groups, otherToken.groups) + return java.util.Objects.equals(username, otherToken.username) + && java.util.Objects.equals(groups, otherToken.groups) && issueTimestamp == otherToken.issueTimestamp && expireTimestamp == otherToken.expireTimestamp; } @Override public int hashCode() { - return Objects.hashCode(getUsername(), + return java.util.Objects.hash(getUsername(), getGroups(), getIssueTimestamp(), getExpireTimestamp()); @@ -162,7 +162,7 @@ public int hashCode() { @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("username", username) .add("tokenType", identifierType) .add("groups", groups) diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/authorization/AccessControllerClassLoader.java b/cdap-security/src/main/java/io/cdap/cdap/security/authorization/AccessControllerClassLoader.java index d38dd6329120..7f90fa3ccbec 100644 --- a/cdap-security/src/main/java/io/cdap/cdap/security/authorization/AccessControllerClassLoader.java +++ b/cdap-security/src/main/java/io/cdap/cdap/security/authorization/AccessControllerClassLoader.java @@ -156,7 +156,7 @@ public byte[] rewriteClass(String className, InputStream input) throws IOExcepti ClassReader cr = new ClassReader(input); ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_FRAMES); - cr.accept(new ClassVisitor(Opcodes.ASM7, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM9, cw) { private String superName; @@ -232,7 +232,7 @@ public void visitEnd() { */ private MethodVisitor rewriteMethod(int access, String name, String descriptor, MethodVisitor mv) { - return new FinallyAdapter(Opcodes.ASM7, mv, access, name, descriptor) { + return new FinallyAdapter(Opcodes.ASM9, mv, access, name, descriptor) { int currentThread; int oldClassLoader; diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/authorization/AccessControllerInstantiator.java b/cdap-security/src/main/java/io/cdap/cdap/security/authorization/AccessControllerInstantiator.java index 9b52be01fdde..11248abe3b7a 100644 --- a/cdap-security/src/main/java/io/cdap/cdap/security/authorization/AccessControllerInstantiator.java +++ b/cdap-security/src/main/java/io/cdap/cdap/security/authorization/AccessControllerInstantiator.java @@ -311,7 +311,13 @@ public void close() throws IOException { } catch (Throwable t) { LOG.warn("Failed to destroy accessController.", t); } finally { - Closeables.closeQuietly(accessControllerClassLoader); + if (accessControllerClassLoader instanceof AutoCloseable) { + try { + ((AutoCloseable) accessControllerClassLoader).close(); + } catch (Exception e) { + // Ignore + } + } } } } diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/runtime/AuthenticationServerMain.java b/cdap-security/src/main/java/io/cdap/cdap/security/runtime/AuthenticationServerMain.java index 85399509197a..581d10fd17d7 100644 --- a/cdap-security/src/main/java/io/cdap/cdap/security/runtime/AuthenticationServerMain.java +++ b/cdap-security/src/main/java/io/cdap/cdap/security/runtime/AuthenticationServerMain.java @@ -87,7 +87,7 @@ public void start() throws Exception { + "ZooKeeper quorum settings are correct in " + "cdap-site.xml. Currently configured as: %s", zkClientService.getConnectString())); - authServer.startAndWait(); + authServer.startAsync().awaitRunning(); } catch (Exception e) { Throwable rootCause = Throwables.getRootCause(e); if (rootCause instanceof ServiceBindException) { diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/server/ExternalAuthenticationServer.java b/cdap-security/src/main/java/io/cdap/cdap/security/server/ExternalAuthenticationServer.java index 2edf817d29c8..844c6cafc437 100644 --- a/cdap-security/src/main/java/io/cdap/cdap/security/server/ExternalAuthenticationServer.java +++ b/cdap-security/src/main/java/io/cdap/cdap/security/server/ExternalAuthenticationServer.java @@ -294,7 +294,7 @@ private Map getAuthHandlerConfigs(Configuration configuration) { } @Override - protected Executor executor(State state) { + protected Executor executor() { final AtomicInteger id = new AtomicInteger(); //noinspection NullableProblems diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/server/GrantAccessToken.java b/cdap-security/src/main/java/io/cdap/cdap/security/server/GrantAccessToken.java index 96c57e21b042..d8e1f297b677 100644 --- a/cdap-security/src/main/java/io/cdap/cdap/security/server/GrantAccessToken.java +++ b/cdap-security/src/main/java/io/cdap/cdap/security/server/GrantAccessToken.java @@ -73,7 +73,7 @@ public GrantAccessToken(TokenManager tokenManager, public void init() { // TokenManager may have already been started in AbstractServiceMain if internal auth is enabled. if (!tokenManager.isRunning()) { - tokenManager.start(); + tokenManager.startAsync(); } } @@ -81,7 +81,7 @@ public void init() { * Stop the TokenManager. */ public void destroy() { - tokenManager.stop(); + tokenManager.stopAsync(); } /** diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/server/LdapAuthenticationHandler.java b/cdap-security/src/main/java/io/cdap/cdap/security/server/LdapAuthenticationHandler.java index c98cb26b481a..c220b859d910 100644 --- a/cdap-security/src/main/java/io/cdap/cdap/security/server/LdapAuthenticationHandler.java +++ b/cdap-security/src/main/java/io/cdap/cdap/security/server/LdapAuthenticationHandler.java @@ -16,7 +16,7 @@ package io.cdap.cdap.security.server; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; import io.cdap.cdap.common.conf.Constants; import java.util.HashMap; @@ -61,9 +61,9 @@ public AppConfigurationEntry[] getAppConfigurationEntry(String s) { String ldapsVerifyCertificate = handlerProps.get("ldapsVerifyCertificate"); String useLdaps = handlerProps.get("useLdaps"); - if (Boolean.parseBoolean(Objects.firstNonNull(useLdaps, "false"))) { + if (Boolean.parseBoolean(MoreObjects.firstNonNull(useLdaps, "false"))) { ldapSSLVerifyCertificate = Boolean.parseBoolean( - Objects.firstNonNull(ldapsVerifyCertificate, "true")); + MoreObjects.firstNonNull(ldapsVerifyCertificate, "true")); } return new AppConfigurationEntry[]{ diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/store/DefaultSecureStoreService.java b/cdap-security/src/main/java/io/cdap/cdap/security/store/DefaultSecureStoreService.java index 268b3cb93b59..b8971fdc113a 100644 --- a/cdap-security/src/main/java/io/cdap/cdap/security/store/DefaultSecureStoreService.java +++ b/cdap-security/src/main/java/io/cdap/cdap/security/store/DefaultSecureStoreService.java @@ -145,11 +145,11 @@ public final void delete(String namespace, String name) throws Exception { @Override protected void startUp() throws Exception { - secureStoreService.startAndWait(); + secureStoreService.startAsync().awaitRunning(); } @Override protected void shutDown() throws Exception { - secureStoreService.stopAndWait(); + secureStoreService.stopAsync().awaitTerminated(); } } diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/tools/AccessTokenGeneratorService.java b/cdap-security/src/main/java/io/cdap/cdap/security/tools/AccessTokenGeneratorService.java index a6491022cfb6..4b5291295577 100644 --- a/cdap-security/src/main/java/io/cdap/cdap/security/tools/AccessTokenGeneratorService.java +++ b/cdap-security/src/main/java/io/cdap/cdap/security/tools/AccessTokenGeneratorService.java @@ -109,7 +109,7 @@ public void stop() { } catch (Exception e) { LOG.warn("Exception when stopping AccessTokenGeneratorService", e); } - handler.tokenManager.stopAndWait(); + handler.tokenManager.stopAsync().awaitTerminated(); } @Override diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/zookeeper/SharedResourceCache.java b/cdap-security/src/main/java/io/cdap/cdap/security/zookeeper/SharedResourceCache.java index c87e464c6b8a..91f20ec6264e 100644 --- a/cdap-security/src/main/java/io/cdap/cdap/security/zookeeper/SharedResourceCache.java +++ b/cdap-security/src/main/java/io/cdap/cdap/security/zookeeper/SharedResourceCache.java @@ -24,6 +24,7 @@ import com.google.common.collect.Sets; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; import io.cdap.cdap.common.io.Codec; import io.cdap.cdap.common.zookeeper.ZKExtOperations; @@ -126,7 +127,7 @@ public void onFailure(Throwable t) { LOG.error("Failed to get data for child node {}", nodeName, t); listeners.notifyError(nodeName, t); } - }); + }, MoreExecutors.directExecutor()); LOG.debug("Added future for {}", child); } } @@ -192,7 +193,7 @@ public void onFailure(Throwable t) { listeners.notifyError(name, t); completion.setException(t); } - } + }, MoreExecutors.directExecutor() ); // Block until it is done @@ -228,7 +229,7 @@ public void onFailure(Throwable t) { LOG.error("Failed to remove znode {}", znode, t); listeners.notifyError(name, t); } - }); + }, MoreExecutors.directExecutor()); } /** @@ -348,7 +349,7 @@ public void onSuccess(NodeData result) { public void onFailure(Throwable t) { resourceCallback.onFailure(t); } - }); + }, MoreExecutors.directExecutor()); } private class ZKWatcher implements Watcher { diff --git a/cdap-security/src/test/java/io/cdap/cdap/security/auth/DistributedKeyManagerTest.java b/cdap-security/src/test/java/io/cdap/cdap/security/auth/DistributedKeyManagerTest.java index c37de574bf86..ce3a27c258a6 100644 --- a/cdap-security/src/test/java/io/cdap/cdap/security/auth/DistributedKeyManagerTest.java +++ b/cdap-security/src/test/java/io/cdap/cdap/security/auth/DistributedKeyManagerTest.java @@ -117,8 +117,8 @@ public void testKeyDistribution() throws Exception { new TestingTokenManager(manager1, injector1.getInstance(UserIdentityCodec.class)); TestingTokenManager tokenManager2 = new TestingTokenManager(manager2, injector2.getInstance(UserIdentityCodec.class)); - tokenManager1.startAndWait(); - tokenManager2.startAndWait(); + tokenManager1.startAsync().awaitRunning(); + tokenManager2.startAsync().awaitRunning(); long now = System.currentTimeMillis(); UserIdentity ident1 = new UserIdentity("testuser", UserIdentity.IdentifierType.EXTERNAL, @@ -136,8 +136,8 @@ public void testKeyDistribution() throws Exception { assertEquals(token1.getIdentifier().getGroups(), token2.getIdentifier().getGroups()); assertEquals(token1, token2); - tokenManager1.stopAndWait(); - tokenManager2.stopAndWait(); + tokenManager1.stopAsync().awaitTerminated(); + tokenManager2.stopAsync().awaitTerminated(); } @Test @@ -160,21 +160,21 @@ protected ImmutablePair> getTokenManagerAndCode DistributedKeyManager keyManager = getKeyManager(injector1, true); TokenManager tokenManager = new TokenManager(keyManager, injector1.getInstance(UserIdentityCodec.class)); - tokenManager.startAndWait(); + tokenManager.startAsync().awaitRunning(); return new ImmutablePair<>(tokenManager, injector1.getInstance(AccessTokenCodec.class)); } private DistributedKeyManager getKeyManager(Injector injector, boolean expectLeader) throws Exception { ZKClientService zk = injector.getInstance(ZKClientService.class); - zk.startAndWait(); + zk.startAsync().awaitRunning(); WaitableDistributedKeyManager keyManager = new WaitableDistributedKeyManager(injector.getInstance(CConfiguration.class), injector.getInstance(Key.get(new TypeLiteral>() { })), zk); - keyManager.startAndWait(); + keyManager.startAsync().awaitRunning(); if (expectLeader) { Tasks.waitFor(true, () -> keyManager.getCurrentKey() != null, 5L, TimeUnit.SECONDS); } diff --git a/cdap-security/src/test/java/io/cdap/cdap/security/auth/FileBasedTokenManagerTest.java b/cdap-security/src/test/java/io/cdap/cdap/security/auth/FileBasedTokenManagerTest.java index e535d834b1b0..de26d1a105ff 100644 --- a/cdap-security/src/test/java/io/cdap/cdap/security/auth/FileBasedTokenManagerTest.java +++ b/cdap-security/src/test/java/io/cdap/cdap/security/auth/FileBasedTokenManagerTest.java @@ -60,7 +60,7 @@ protected ImmutablePair> getTokenManagerAndCode new FileBasedCoreSecurityModule(), new InMemoryDiscoveryModule()); TokenManager tokenManager = injector.getInstance(TokenManager.class); - tokenManager.startAndWait(); + tokenManager.startAsync().awaitRunning(); Codec tokenCodec = injector.getInstance(AccessTokenCodec.class); return new ImmutablePair<>(tokenManager, tokenCodec); } @@ -79,14 +79,14 @@ public void testFileBasedKey() throws Exception { new ConfigModule(cConf), new FileBasedCoreSecurityModule(), new InMemoryDiscoveryModule()).getInstance(TokenManager.class); - tokenManager.startAndWait(); + tokenManager.startAsync().awaitRunning(); TokenManager tokenManager2 = Guice.createInjector( new IOModule(), new ConfigModule(cConf), new FileBasedCoreSecurityModule(), new InMemoryDiscoveryModule()).getInstance(TokenManager.class); - tokenManager2.startAndWait(); + tokenManager2.startAsync().awaitRunning(); Assert.assertNotSame("ERROR: Both token managers refer to the same object.", tokenManager, tokenManager2); @@ -129,7 +129,7 @@ public void testKeyUpdate() throws Exception { keyFile.setLastModified(System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(10)); try { - keyManager.startAndWait(); + keyManager.startAsync().awaitRunning(); // Upon the key manager starts, the current key should be the same as the one from the key file. Assert.assertEquals(keyIdentifier, keyManager.currentKey); @@ -142,7 +142,7 @@ public void testKeyUpdate() throws Exception { Tasks.waitFor(keyIdentifier, () -> keyManager.currentKey, 20, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); } finally { - keyManager.stopAndWait(); + keyManager.stopAsync().awaitTerminated(); } } diff --git a/cdap-security/src/test/java/io/cdap/cdap/security/auth/TestInMemoryTokenManager.java b/cdap-security/src/test/java/io/cdap/cdap/security/auth/TestInMemoryTokenManager.java index 2c61c976fdbf..edb82072f922 100644 --- a/cdap-security/src/test/java/io/cdap/cdap/security/auth/TestInMemoryTokenManager.java +++ b/cdap-security/src/test/java/io/cdap/cdap/security/auth/TestInMemoryTokenManager.java @@ -36,7 +36,7 @@ protected ImmutablePair> getTokenManagerAndCode Injector injector = Guice.createInjector(new IOModule(), new CoreSecurityRuntimeModule().getStandaloneModules(), new ConfigModule(), new InMemoryDiscoveryModule()); TokenManager tokenManager = injector.getInstance(TokenManager.class); - tokenManager.startAndWait(); + tokenManager.startAsync().awaitRunning(); Codec tokenCodec = injector.getInstance(AccessTokenCodec.class); return new ImmutablePair<>(tokenManager, tokenCodec); } diff --git a/cdap-security/src/test/java/io/cdap/cdap/security/auth/TestTokenManager.java b/cdap-security/src/test/java/io/cdap/cdap/security/auth/TestTokenManager.java index a4a29eb1eb54..9f3e80357d88 100644 --- a/cdap-security/src/test/java/io/cdap/cdap/security/auth/TestTokenManager.java +++ b/cdap-security/src/test/java/io/cdap/cdap/security/auth/TestTokenManager.java @@ -42,7 +42,7 @@ public abstract class TestTokenManager { public void testTokenValidation() throws Exception { ImmutablePair> pair = getTokenManagerAndCodec(); TokenManager tokenManager = pair.getFirst(); - tokenManager.startAndWait(); + tokenManager.startAsync().awaitRunning(); Codec tokenCodec = pair.getSecond(); long now = System.currentTimeMillis(); @@ -91,14 +91,14 @@ public void testTokenValidation() throws Exception { // expected } - tokenManager.stopAndWait(); + tokenManager.stopAsync().awaitTerminated(); } @Test public void testTokenSerialization() throws Exception { ImmutablePair> pair = getTokenManagerAndCodec(); TokenManager tokenManager = pair.getFirst(); - tokenManager.startAndWait(); + tokenManager.startAsync().awaitRunning(); Codec tokenCodec = pair.getSecond(); long now = System.currentTimeMillis(); @@ -116,6 +116,6 @@ public void testTokenSerialization() throws Exception { // should be valid since we just signed it tokenManager.validateSecret(token2); - tokenManager.stopAndWait(); + tokenManager.stopAsync().awaitTerminated(); } } diff --git a/cdap-security/src/test/java/io/cdap/cdap/security/server/ExternalAuthenticationServerTestBase.java b/cdap-security/src/test/java/io/cdap/cdap/security/server/ExternalAuthenticationServerTestBase.java index 9e378e10531c..7fcdbc2a8fa2 100644 --- a/cdap-security/src/test/java/io/cdap/cdap/security/server/ExternalAuthenticationServerTestBase.java +++ b/cdap-security/src/test/java/io/cdap/cdap/security/server/ExternalAuthenticationServerTestBase.java @@ -124,14 +124,14 @@ protected void configure() { startExternalAuthenticationServer(); - server.startAndWait(); + server.startAsync().awaitRunning(); LOG.info("Auth server running on address {}", server.getSocketAddress()); TimeUnit.SECONDS.sleep(3); } protected void tearDown() throws Exception { stopExternalAuthenticationServer(); - server.stopAndWait(); + server.stopAsync().awaitTerminated(); // Clear any security properties for zookeeper. System.clearProperty(Constants.External.Zookeeper.ENV_AUTH_PROVIDER_1); Configuration.setConfiguration(null); diff --git a/cdap-security/src/test/java/io/cdap/cdap/security/store/secretmanager/SecretManagerSecureStoreServiceTest.java b/cdap-security/src/test/java/io/cdap/cdap/security/store/secretmanager/SecretManagerSecureStoreServiceTest.java index b6d2cb829b02..709d7889093a 100644 --- a/cdap-security/src/test/java/io/cdap/cdap/security/store/secretmanager/SecretManagerSecureStoreServiceTest.java +++ b/cdap-security/src/test/java/io/cdap/cdap/security/store/secretmanager/SecretManagerSecureStoreServiceTest.java @@ -48,12 +48,12 @@ public static void setUp() throws Exception { namespaceClient.create(namespaceMeta); secureStoreService = new SecretManagerSecureStoreService(namespaceClient, new MockSecretManagerContext(), "mock", new MockSecretManager()); - secureStoreService.startAndWait(); + secureStoreService.startAsync().awaitRunning(); } @AfterClass public static void cleanUp() { - secureStoreService.stopAndWait(); + secureStoreService.stopAsync().awaitTerminated(); } @Test diff --git a/cdap-security/src/test/java/io/cdap/cdap/security/zookeeper/SharedResourceCacheTest.java b/cdap-security/src/test/java/io/cdap/cdap/security/zookeeper/SharedResourceCacheTest.java index 40fa80649cb4..6483a27b39e7 100644 --- a/cdap-security/src/test/java/io/cdap/cdap/security/zookeeper/SharedResourceCacheTest.java +++ b/cdap-security/src/test/java/io/cdap/cdap/security/zookeeper/SharedResourceCacheTest.java @@ -84,7 +84,7 @@ public void testCache() throws Exception { // create 2 cache instances ZKClientService zkClient1 = injector1.getInstance(ZKClientService.class); - zkClient1.startAndWait(); + zkClient1.startAsync().awaitRunning(); SharedResourceCache cache1 = new SharedResourceCache<>(zkClient1, new StringCodec(), parentNode, acls); cache1.init(); @@ -95,7 +95,7 @@ public void testCache() throws Exception { cache1.put(key1, value1); ZKClientService zkClient2 = injector2.getInstance(ZKClientService.class); - zkClient2.startAndWait(); + zkClient2.startAsync().awaitRunning(); SharedResourceCache cache2 = new SharedResourceCache<>(zkClient2, new StringCodec(), parentNode, acls); cache2.init(); @@ -194,8 +194,8 @@ private void waitForEntry(SharedResourceCache cache, String key, String String value = cache.get(key); boolean isPresent = expectedValue.equals(value); - Stopwatch watch = new Stopwatch().start(); - while (!isPresent && watch.elapsedTime(TimeUnit.MILLISECONDS) < timeToWaitMillis) { + Stopwatch watch = Stopwatch.createStarted(); + while (!isPresent && watch.elapsed(TimeUnit.MILLISECONDS) < timeToWaitMillis) { TimeUnit.MILLISECONDS.sleep(200); value = cache.get(key); isPresent = expectedValue.equals(value); diff --git a/cdap-source-control/pom.xml b/cdap-source-control/pom.xml index ae1128fb1de5..775833893420 100644 --- a/cdap-source-control/pom.xml +++ b/cdap-source-control/pom.xml @@ -110,4 +110,18 @@ test + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + diff --git a/cdap-source-control/src/main/java/io/cdap/cdap/sourcecontrol/RepositoryManager.java b/cdap-source-control/src/main/java/io/cdap/cdap/sourcecontrol/RepositoryManager.java index 919ad38fac36..324d0b7348e1 100644 --- a/cdap-source-control/src/main/java/io/cdap/cdap/sourcecontrol/RepositoryManager.java +++ b/cdap-source-control/src/main/java/io/cdap/cdap/sourcecontrol/RepositoryManager.java @@ -233,7 +233,7 @@ public > CommitResult commitAndPush( CommitMeta commitMeta, Collection filesChanged, BiFunction hashConsumer) throws NoChangesToPushException, GitAPIException { validateInitialized(); - final Stopwatch stopwatch = new Stopwatch().start(); + final Stopwatch stopwatch = Stopwatch.createStarted(); // if the status is clean skip Status preStageStatus = git.status().call(); @@ -275,7 +275,7 @@ public > CommitResult commitAndPush( metricsContext.event( SourceControlManagement.COMMIT_PUSH_LATENCY_MILLIS, - stopwatch.stop().elapsedTime(TimeUnit.MILLISECONDS)); + stopwatch.stop().elapsed(TimeUnit.MILLISECONDS)); return new CommitResult<>(commit.getName(), output); } @@ -318,10 +318,10 @@ public String cloneRemote() .setBranch(branch); } - final Stopwatch stopwatch = new Stopwatch().start(); + final Stopwatch stopwatch = Stopwatch.createStarted(); git = command.call(); final long cloneTimeMillis = stopwatch.stop() - .elapsedTime(TimeUnit.MILLISECONDS); + .elapsed(TimeUnit.MILLISECONDS); // Record the repository size metric. try { diff --git a/cdap-source-control/src/main/java/io/cdap/cdap/sourcecontrol/worker/SourceControlTask.java b/cdap-source-control/src/main/java/io/cdap/cdap/sourcecontrol/worker/SourceControlTask.java index b2a3066f10a8..09e9716ee27e 100644 --- a/cdap-source-control/src/main/java/io/cdap/cdap/sourcecontrol/worker/SourceControlTask.java +++ b/cdap-source-control/src/main/java/io/cdap/cdap/sourcecontrol/worker/SourceControlTask.java @@ -56,7 +56,7 @@ abstract class SourceControlTask implements RunnableTask { @Override public void run(RunnableTaskContext context) throws Exception { - inMemoryOperationRunner.startAndWait(); + inMemoryOperationRunner.startAsync().awaitRunning(); doRun(context); } diff --git a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkProgramRunner.java b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkProgramRunner.java index 9647532e75f6..9878de70c0c3 100644 --- a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkProgramRunner.java +++ b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkProgramRunner.java @@ -246,7 +246,7 @@ public ProgramController run(Program program, ProgramOptions options) { LOG.debug("Starting Spark Job. Context: {}", runtimeContext); if (isLocal || UserGroupInformation.isSecurityEnabled()) { - sparkRuntimeService.start(); + sparkRuntimeService.startAsync(); } else { ProgramRunners.startAsUser(cConf.get(Constants.CFG_HDFS_USER), sparkRuntimeService); } @@ -277,7 +277,11 @@ public void run() { // for shutting down threads. During shutdown, there are new classes being loaded. Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); if (classLoader instanceof Closeable) { - Closeables.closeQuietly((Closeable) classLoader); + try { + ((Closeable) classLoader).close(); + } catch (IOException e) { + // ignore + } LOG.trace("Closed SparkProgramRunner ClassLoader {}", classLoader); } } diff --git a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkProgramRuntimeProvider.java b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkProgramRuntimeProvider.java index cd451d3f5836..d83d156d5c5d 100644 --- a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkProgramRuntimeProvider.java +++ b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkProgramRuntimeProvider.java @@ -141,7 +141,11 @@ public ProgramRunner createProgramRunner(ProgramType type, Mode mode, Injector i SparkProgramRunner.class.getName(), classLoader); } catch (Throwable t) { // If there is any exception, close the classloader - Closeables.closeQuietly(classLoader); + try { + classLoader.close(); + } catch (IOException e) { + // ignore + } throw t; } } catch (IOException e) { diff --git a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkRuntimeContext.java b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkRuntimeContext.java index 98cbd02d573d..dca655e53a46 100644 --- a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkRuntimeContext.java +++ b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkRuntimeContext.java @@ -16,7 +16,7 @@ package io.cdap.cdap.app.runtime.spark; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import com.google.common.io.Closeables; @@ -115,7 +115,11 @@ true, metricsCollectionService, createMetricsTags(workflowProgramInfo), @Override public void close() { super.close(); - Closeables.closeQuietly(closeable); + try { + closeable.close(); + } catch (java.io.IOException e) { + // ignore + } } @Override @@ -264,7 +268,7 @@ public LocationFactory getLocationFactory() { @Override public String toString() { - return Objects.toStringHelper(SparkRuntimeContext.class) + return MoreObjects.toStringHelper(SparkRuntimeContext.class) .add("id", getProgram().getId()) .add("runId", getRunId()) .toString(); diff --git a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkRuntimeContextProvider.java b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkRuntimeContextProvider.java index 9d73eb074425..f62d6bd1ae65 100644 --- a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkRuntimeContextProvider.java +++ b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkRuntimeContextProvider.java @@ -214,7 +214,7 @@ private static synchronized SparkRuntimeContext createIfNotExists() { // For spark running natively on k8s, we may need to initialize the TokenManager for internal identity. if (clusterMode == ClusterMode.ON_PREMISE && SecurityUtil.isInternalAuthEnabled(cConf)) { TokenManager tokenManager = injector.getInstance(TokenManager.class); - tokenManager.startAndWait(); + tokenManager.startAsync().awaitRunning(); } SystemArguments.setLogLevel(programOptions.getUserArguments(), logAppenderInitializer); @@ -240,7 +240,7 @@ private static synchronized SparkRuntimeContext createIfNotExists() { coreServices.add(serviceAnnouncer); for (Service coreService : coreServices) { - coreService.startAndWait(); + coreService.startAsync().awaitRunning(); } AtomicBoolean closed = new AtomicBoolean(); @@ -254,7 +254,7 @@ private static synchronized SparkRuntimeContext createIfNotExists() { // Stop all services. Reverse the order. for (Service service : (Iterable) coreServices::descendingIterator) { try { - service.stopAndWait(); + service.stopAsync().awaitTerminated(); } catch (Exception e) { LOG.warn("Exception raised when stopping service {} during program termination.", service, e); } diff --git a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkRuntimeService.java b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkRuntimeService.java index cdbb2e9fc613..88473303d9de 100644 --- a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkRuntimeService.java +++ b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkRuntimeService.java @@ -205,7 +205,7 @@ public void destroy() { } @Override - protected String getServiceName() { + protected String serviceName() { return "Spark - " + runtimeContext.getSparkSpecification().getName(); } @@ -285,7 +285,7 @@ protected void startUp() throws Exception { // In case of spark-on-k8s, artifactFetcherService is used by spark-drivers for fetching artifacts bundle. Location location = createBundle(new File("./artifacts").getAbsoluteFile().toPath()); artifactFetcherService = new ArtifactFetcherService(cConf, location, commonNettyHttpServiceFactory); - artifactFetcherService.startAndWait(); + artifactFetcherService.startAsync().awaitRunning(); } } else if (isLocal) { @@ -472,7 +472,7 @@ protected void shutDown() throws Exception { } finally { try { if (artifactFetcherService != null) { - artifactFetcherService.stopAndWait(); + artifactFetcherService.stopAsync().awaitTerminated(); } } finally { cleanupTask.run(); @@ -511,7 +511,20 @@ protected void triggerShutdown() { */ public ListenableFuture stop(long timeout, TimeUnit timeoutUnit) { gracefulTimeoutMillis = timeoutUnit.toMillis(timeout); - return stop(); + stopAsync(); + com.google.common.util.concurrent.SettableFuture future = + com.google.common.util.concurrent.SettableFuture.create(); + addListener(new com.google.common.util.concurrent.Service.Listener() { + @Override + public void terminated(State from) { + future.set(State.TERMINATED); + } + @Override + public void failed(State from, Throwable failure) { + future.setException(failure); + } + }, org.apache.twill.common.Threads.SAME_THREAD_EXECUTOR); + return future; } @Override @@ -856,9 +869,10 @@ private URI getPySparkScript(File tempDir) throws IOException, URISyntaxExceptio hConf.set(String.format("fs.%s.impl.disable.cache", scriptURI.getScheme()), "true"); try ( FileSystem fs = FileSystem.get(scriptURI, hConf); - InputStream is = fs.open(new Path(scriptURI)) + InputStream is = fs.open(new Path(scriptURI)); + FileOutputStream os = new FileOutputStream(pythonFile) ) { - ByteStreams.copy(is, Files.newOutputStreamSupplier(pythonFile)); + ByteStreams.copy(is, os); return pythonFile.toURI(); } } catch (IOException e) { @@ -867,8 +881,9 @@ private URI getPySparkScript(File tempDir) throws IOException, URISyntaxExceptio } // Last resort, turn the URI to URL and try to copy from it. - try (InputStream is = scriptURI.toURL().openStream()) { - ByteStreams.copy(is, Files.newOutputStreamSupplier(pythonFile)); + try (InputStream is = scriptURI.toURL().openStream(); + FileOutputStream os = new FileOutputStream(pythonFile)) { + ByteStreams.copy(is, os); } return pythonFile.toURI(); } diff --git a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkRuntimeUtils.java b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkRuntimeUtils.java index 72ee04539959..719794c3e5e0 100644 --- a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkRuntimeUtils.java +++ b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkRuntimeUtils.java @@ -199,7 +199,7 @@ public static SparkProgramCompletion initSparkMain() { // (via SparkMainWraper). // We use a service listener so that it can handle all cases. final CountDownLatch secStopLatch = new CountDownLatch(1); - driverService.addListener(new ServiceListenerAdapter() { + driverService.addListener(new Service.Listener() { @Override public void stopping(Service.State from) { @@ -240,7 +240,7 @@ private void handleStopped() { } }, Threads.SAME_THREAD_EXECUTOR); - driverService.startAndWait(); + driverService.startAsync().awaitRunning(); return new SparkProgramCompletion() { @Override public void completed() { @@ -267,7 +267,7 @@ private void handleCompleted(@Nullable Throwable t) { delegate.completedWithException(t); } } else { - driverService.stop(); + driverService.stopAsync(); } // Wait for the spark execution context to stop to make sure everything related diff --git a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkTransactionClient.java b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkTransactionClient.java index 8a2959d65909..b1271cfd8b72 100644 --- a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkTransactionClient.java +++ b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkTransactionClient.java @@ -70,10 +70,10 @@ public Transaction getTransaction(int stageId, long timeout, TransactionFailureException { long timeoutMillis = Math.max(0L, timeUnit.toMillis(timeout) - txPollIntervalMillis); - Stopwatch stopwatch = new Stopwatch().start(); + Stopwatch stopwatch = Stopwatch.createStarted(); Transaction transaction = getTransaction(stageId); - while (transaction == null && stopwatch.elapsedMillis() < timeoutMillis) { + while (transaction == null && stopwatch.elapsed(TimeUnit.MILLISECONDS) < timeoutMillis) { TimeUnit.MILLISECONDS.sleep(txPollIntervalMillis); transaction = getTransaction(stageId); } diff --git a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/classloader/SparkClassRewriter.java b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/classloader/SparkClassRewriter.java index 70e0de1238a9..de3c2438cd1c 100644 --- a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/classloader/SparkClassRewriter.java +++ b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/classloader/SparkClassRewriter.java @@ -263,7 +263,7 @@ private byte[] rewriteOutputMetrics(Type type, InputStream byteCodeStream) throw ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_FRAMES); Method setBytesWrittenMethod = new Method("setBytesWritten", Type.VOID_TYPE, new Type[]{Type.LONG_TYPE}); - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM9, cw) { @Override public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { @@ -272,7 +272,7 @@ public MethodVisitor visitMethod(int access, String name, String descriptor, return mv; } - return new AdviceAdapter(Opcodes.ASM5, mv, access, name, descriptor) { + return new AdviceAdapter(Opcodes.ASM9, mv, access, name, descriptor) { private final Label skipZeroLabel = newLabel(); @@ -357,11 +357,11 @@ private byte[] rewriteDStreamGraph(InputStream byteCodeStream) throws IOExceptio ClassReader cr = new ClassReader(byteCodeStream); ClassWriter cw = new ClassWriter(0); - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM9, cw) { @Override public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) { MethodVisitor mv = super.visitMethod(access, name, desc, signature, exceptions); - return new MethodVisitor(Opcodes.ASM5, mv) { + return new MethodVisitor(Opcodes.ASM9, mv) { @Override public void visitMethodInsn(int opcode, String owner, String name, String desc, boolean itf) { super.visitMethodInsn(opcode, owner, name, desc, itf); @@ -408,7 +408,7 @@ public void onMethodExit(String name, String desc, GeneratorAdapter generatorAda ClassReader cr = new ClassReader(bytecode); ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_MAXS); Method closeMethod = new Method("close", Type.VOID_TYPE, new Type[0]); - cr.accept(new ClassVisitor(Opcodes.ASM7, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM9, cw) { @Override public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { @@ -443,7 +443,7 @@ private byte[] rewriteWritAheadLogHandler(InputStream byteCodeStream) throws IOE Method shutdownMethod = new Method("shutdown", Type.VOID_TYPE, new Type[0]); String executorClass = "scala/concurrent/ExecutionContextExecutorService"; - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM9, cw) { @Override public MethodVisitor visitMethod(int access, String name, String descriptor, @@ -454,7 +454,7 @@ public MethodVisitor visitMethod(int access, String name, String descriptor, return mv; } - return new MethodVisitor(Opcodes.ASM7, mv) { + return new MethodVisitor(Opcodes.ASM9, mv) { @Override public void visitMethodInsn(int opcode, String owner, String name, String descriptor, boolean isInterface) { if (Opcodes.INVOKEINTERFACE == opcode @@ -519,7 +519,7 @@ private byte[] rewriteExecutorClassLoader(InputStream byteCodeStream) throws IOE new Type[] { Type.getType(String.class) }); resourceMethods.put(method, null); - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM9, cw) { private boolean hasParentLoader; private boolean rewriteInit; @@ -544,7 +544,7 @@ public MethodVisitor visitMethod(int access, String name, String desc, String si if (!rewriteInit || !"".equals(name)) { return mv; } - return new GeneratorAdapter(Opcodes.ASM5, mv, access, name, desc) { + return new GeneratorAdapter(Opcodes.ASM9, mv, access, name, desc) { @Override public void visitMethodInsn(int opcode, String owner, String name, String desc, boolean itf) { @@ -673,7 +673,7 @@ private byte[] rewriteConstructor(final Type classType, InputStream byteCodeStre ClassReader cr = new ClassReader(byteCodeStream); ClassWriter cw = new ClassWriter(0); - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM9, cw) { @Override public MethodVisitor visitMethod(int access, final String name, @@ -686,7 +686,7 @@ public MethodVisitor visitMethod(int access, final String name, return mv; } - return new AdviceAdapter(Opcodes.ASM5, mv, access, name, desc) { + return new AdviceAdapter(Opcodes.ASM9, mv, access, name, desc) { boolean calledThis; @@ -739,11 +739,11 @@ private byte[] rewriteSetProperties(InputStream byteCodeStream) throws IOExcepti ClassReader cr = new ClassReader(byteCodeStream); ClassWriter cw = new ClassWriter(0); - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM9, cw) { @Override public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) { MethodVisitor mv = super.visitMethod(access, name, desc, signature, exceptions); - return new MethodVisitor(Opcodes.ASM5, mv) { + return new MethodVisitor(Opcodes.ASM9, mv) { @Override public void visitMethodInsn(int opcode, String owner, String name, String desc, boolean itf) { // If we see a call to System.setProperty, change it to SparkRuntimeEnv.setProperty @@ -777,7 +777,7 @@ private byte[] rewriteTempFileNameForCheckpoint(InputStream byteCodeStream) thro ClassReader cr = new ClassReader(byteCodeStream); ClassWriter cw = new ClassWriter(0); - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM9, cw) { boolean hasCheckpointTimeField; @@ -798,7 +798,7 @@ public MethodVisitor visitMethod(int access, String name, String desc, String si MethodVisitor mv = super.visitMethod(access, name, desc, signature, exceptions); - return new GeneratorAdapter(Opcodes.ASM5, mv, access, name, desc) { + return new GeneratorAdapter(Opcodes.ASM9, mv, access, name, desc) { boolean tempStringAddedToStack; final Method hadoopPathConstructorMethod = @@ -882,7 +882,7 @@ private byte[] rewritePythonRunner(InputStream byteCodeStream) throws IOExceptio // Intercept the static void main(String[] args) method. final Method mainMethod = new Method("main", Type.VOID_TYPE, new Type[] { Type.getType(String[].class) }); - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM9, cw) { @Override public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) { MethodVisitor mv = super.visitMethod(access, name, desc, signature, exceptions); @@ -901,7 +901,7 @@ public MethodVisitor visitMethod(int access, String name, String desc, String si // [local_mode] throw (t instance of SparkUserAppException) ? new RuntimeException(t) : t; // [distributed_mode] throw t; // } - return new AdviceAdapter(Opcodes.ASM5, mv, access, name, desc) { + return new AdviceAdapter(Opcodes.ASM9, mv, access, name, desc) { final Type sparkUserAppExceptionType = Type.getObjectType("org/apache/spark/SparkUserAppException"); final Type throwableType = Type.getType(Throwable.class); @@ -983,7 +983,7 @@ private byte[] rewritePythonRunnerCompanion(InputStream byteCodeStream) throws I ClassReader cr = new ClassReader(byteCodeStream); ClassWriter cw = new ClassWriter(0); - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM9, cw) { @Override public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) { // Intercept all methods @@ -1003,7 +1003,7 @@ private byte[] rewritePythonWorkerFactory(InputStream byteCodeStream) throws IOE ClassReader cr = new ClassReader(byteCodeStream); ClassWriter cw = new ClassWriter(0); - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM9, cw) { @Override public MethodVisitor visitMethod(int access, String name, @@ -1013,7 +1013,7 @@ public MethodVisitor visitMethod(int access, String name, access, name, desc, distributed); final GeneratorAdapter adapter = new GeneratorAdapter(mv, access, name, desc); - return new MethodVisitor(Opcodes.ASM5, mv) { + return new MethodVisitor(Opcodes.ASM9, mv) { @Override public void visitFieldInsn(int opcode, String owner, String name, String desc) { @@ -1077,7 +1077,7 @@ private byte[] rewriteAkkaRemoting(InputStream byteCodeStream) throws IOExceptio ClassReader cr = new ClassReader(byteCodeStream); ClassWriter cw = new ClassWriter(0); - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM9, cw) { @Override public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) { // Call super so that the method signature is registered with the ClassWriter (parent) @@ -1088,7 +1088,7 @@ public MethodVisitor visitMethod(int access, String name, String desc, String si return mv; } - return new MethodVisitor(Opcodes.ASM5, mv) { + return new MethodVisitor(Opcodes.ASM9, mv) { @Override public void visitMethodInsn(int opcode, String owner, String name, String desc, boolean itf) { // Detect if it is making call "import scala.concurrent.ExecutionContext.Implicits.global", @@ -1150,7 +1150,7 @@ private byte[] rewriteSparkNetworkClass(InputStream byteCodeStream) throws IOExc // Scan for class type and methods to see if the rewriting is needed final AtomicBoolean rewritten = new AtomicBoolean(false); - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM9, cw) { private Type classType; private boolean isFileRegion; @@ -1252,7 +1252,7 @@ private Type determineAkkaDispatcherReturnType() { final AtomicReference result = new AtomicReference<>(); ClassReader cr = new ClassReader(is); - cr.accept(new ClassVisitor(Opcodes.ASM5) { + cr.accept(new ClassVisitor(Opcodes.ASM9) { @Override public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) { if (name.equals("dispatcher") && Type.getArgumentTypes(desc).length == 0) { @@ -1302,7 +1302,7 @@ private byte[] rewriteClient(InputStream byteCodeStream) throws IOException { ClassReader cr = new ClassReader(byteCodeStream); ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_MAXS); - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM9, cw) { @Override public MethodVisitor visitMethod(final int access, final String name, final String desc, String signature, String[] exceptions) { @@ -1418,7 +1418,7 @@ private static final class OutputRedirectMethodVisitor extends MethodVisitor { private final boolean distributed; OutputRedirectMethodVisitor(MethodVisitor mv, int access, String name, String desc, boolean distributed) { - super(Opcodes.ASM5, mv); + super(Opcodes.ASM9, mv); this.adapter = new GeneratorAdapter(mv, access, name, desc); this.distributed = distributed; } diff --git a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/distributed/SparkContainerLauncher.java b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/distributed/SparkContainerLauncher.java index 5e535872552a..bc6063581410 100644 --- a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/distributed/SparkContainerLauncher.java +++ b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/distributed/SparkContainerLauncher.java @@ -206,7 +206,11 @@ public static void launch(String mainClassName, String[] args, boolean removeMai throw t; } finally { if (sparkRuntimeContext instanceof Closeable) { - Closeables.closeQuietly((Closeable) sparkRuntimeContext); + try { + ((Closeable) sparkRuntimeContext).close(); + } catch (IOException e) { + // ignore + } } } } diff --git a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/distributed/SparkDriverService.java b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/distributed/SparkDriverService.java index 2e9b8fed6dca..478856bd168f 100644 --- a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/distributed/SparkDriverService.java +++ b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/distributed/SparkDriverService.java @@ -107,7 +107,7 @@ protected void startUp() throws Exception { // Schedule the credentials update if necessary if (credentialsUpdater != null) { - credentialsUpdater.startAndWait(); + credentialsUpdater.startAsync().awaitRunning(); } LOG.info("SparkDriverService started."); @@ -167,7 +167,7 @@ protected void shutDown() throws Exception { Thread.interrupted(); try { if (credentialsUpdater != null) { - credentialsUpdater.stopAndWait(); + credentialsUpdater.stopAsync().awaitTerminated(); } } finally { if (completionState.get() == CompletionState.COMPLETED) { @@ -289,7 +289,7 @@ private void heartbeat(SparkExecutionClient client, @Nullable BasicWorkflowToken terminateTs, terminationTimeout); } runtimeContext.setTerminationTime(TimeUnit.SECONDS.toMillis(terminateTs)); - stop(); + stopAsync(); } else { LOG.warn("Ignoring unsupported command {}", command); } diff --git a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/distributed/SparkExecutionService.java b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/distributed/SparkExecutionService.java index feb00560141d..4aa8c8a9d5c6 100644 --- a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/distributed/SparkExecutionService.java +++ b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/distributed/SparkExecutionService.java @@ -160,7 +160,7 @@ protected void shutDown() throws Exception { */ public void shutdownNow() { stopLatch.countDown(); - stopAndWait(); + stopAsync().awaitTerminated(); } /** diff --git a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/distributed/SparkRuntimeSecurityManager.java b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/distributed/SparkRuntimeSecurityManager.java index 2ea7006f33fe..dd1e96ac4522 100644 --- a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/distributed/SparkRuntimeSecurityManager.java +++ b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/distributed/SparkRuntimeSecurityManager.java @@ -66,7 +66,11 @@ public void checkExit(int status) { if (delegate != null) { delegate.checkExit(status); } - Closeables.closeQuietly(closeable); + try { + closeable.close(); + } catch (java.io.IOException e) { + // ignore + } } /** diff --git a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/distributed/k8s/SparkContainerDriverLauncher.java b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/distributed/k8s/SparkContainerDriverLauncher.java index 14a3f4efd218..5419e0753cc7 100644 --- a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/distributed/k8s/SparkContainerDriverLauncher.java +++ b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/distributed/k8s/SparkContainerDriverLauncher.java @@ -168,7 +168,7 @@ public static void main(String[] args) throws Exception { artifactFetcherService = new ArtifactFetcherService(cConf, createBundle(new File(WORKING_DIRECTORY).getAbsoluteFile().toPath()), injector.getInstance(CommonNettyHttpServiceFactory.class)); - artifactFetcherService.startAndWait(); + artifactFetcherService.startAsync().awaitRunning(); SparkContainerLauncher.launch(delegateClass, delegateArgs.toArray(new String[delegateArgs.size()]), false, "k8s"); } diff --git a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/service/DefaultSparkHttpServicePluginContext.java b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/service/DefaultSparkHttpServicePluginContext.java index 52e3192edbc4..90b71f9360c2 100644 --- a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/service/DefaultSparkHttpServicePluginContext.java +++ b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/service/DefaultSparkHttpServicePluginContext.java @@ -91,7 +91,11 @@ public DefaultSparkHttpServicePluginContext() throws IOException { TaskContext.get().addTaskCompletionListener(new TaskCompletionListener() { @Override public void onTaskCompletion(TaskContext context) { - Closeables.closeQuietly(pluginInstantiator); + try { + pluginInstantiator.close(); + } catch (IOException e) { + // ignore + } } }); } @@ -261,7 +265,11 @@ public void readExternal(ObjectInput in) throws IOException, ClassNotFoundExcept @Override public void close() { - Closeables.closeQuietly(pluginInstantiator); + try { + pluginInstantiator.close(); + } catch (IOException e) { + // ignore + } } @Override diff --git a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/submit/DistributedSparkSubmitter.java b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/submit/DistributedSparkSubmitter.java index 18de0aba5da2..fda23dd50569 100644 --- a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/submit/DistributedSparkSubmitter.java +++ b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/submit/DistributedSparkSubmitter.java @@ -113,7 +113,7 @@ protected List beforeSubmit() { } } - sparkExecutionService.startAndWait(); + sparkExecutionService.startAsync().awaitRunning(); SparkRuntimeEnv.setProperty("spark.yarn.appMasterEnv." + SparkRuntimeUtils.CDAP_SPARK_EXECUTION_SERVICE_URI, sparkExecutionService.getBaseURI().toString()); return Collections.emptyList(); @@ -124,7 +124,7 @@ protected void triggerShutdown(long timeout, TimeUnit timeoutTimeUnit) { // Just stop the execution service and block on that. // It will wait until the "completed" call from the Spark driver. sparkExecutionService.setShutdownWaitSeconds(timeoutTimeUnit.toSeconds(timeout)); - sparkExecutionService.stopAndWait(); + sparkExecutionService.stopAsync().awaitTerminated(); } @Override diff --git a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/submit/MasterEnvironmentSparkSubmitter.java b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/submit/MasterEnvironmentSparkSubmitter.java index 60d428b97f31..9d748ab6fffa 100644 --- a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/submit/MasterEnvironmentSparkSubmitter.java +++ b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/submit/MasterEnvironmentSparkSubmitter.java @@ -138,7 +138,7 @@ protected void addMaster(Map configs, ImmutableList.Builder beforeSubmit() throws Exception { - sparkExecutionService.startAndWait(); + sparkExecutionService.startAsync().awaitRunning(); InetSocketAddress socketAddress = sparkExecutionService.getBindAddress(); // use ip instead of hostname, as some environments (like kubernetes) don't work properly with hostname String uri = String.format("http://%s:%d", socketAddress.getAddress().getHostAddress(), socketAddress.getPort()); @@ -164,7 +164,7 @@ protected void triggerShutdown(long timeout, TimeUnit timeoutTimeUnit) { // Just stop the execution service and block on that. // It will wait until the "completed" call from the Spark driver. sparkExecutionService.setShutdownWaitSeconds(timeoutTimeUnit.toSeconds(timeout)); - sparkExecutionService.stopAndWait(); + sparkExecutionService.stopAsync().awaitTerminated(); } @Override diff --git a/cdap-spark-core-base/src/main/scala/io/cdap/cdap/app/runtime/spark/AbstractSparkExecutionContext.scala b/cdap-spark-core-base/src/main/scala/io/cdap/cdap/app/runtime/spark/AbstractSparkExecutionContext.scala index 69e39061eaf1..7648cef6afae 100644 --- a/cdap-spark-core-base/src/main/scala/io/cdap/cdap/app/runtime/spark/AbstractSparkExecutionContext.scala +++ b/cdap-spark-core-base/src/main/scala/io/cdap/cdap/app/runtime/spark/AbstractSparkExecutionContext.scala @@ -113,7 +113,7 @@ abstract class AbstractSparkExecutionContext(sparkClassLoader: SparkClassLoader, private var sparkHttpServiceServer: Option[SparkHttpServiceServer] = None // Start the Spark driver http service - sparkDriveHttpService.startAndWait() + sparkDriveHttpService.startAsync().awaitRunning() // Set the spark.repl.class.uri that points to the http service if spark-repl is present try { @@ -132,13 +132,13 @@ abstract class AbstractSparkExecutionContext(sparkClassLoader: SparkClassLoader, if (!handlers.isEmpty) { val httpServer = new SparkHttpServiceServer( runtimeContext, new DefaultSparkHttpServiceContext(AbstractSparkExecutionContext.this)) - httpServer.startAndWait() + httpServer.startAsync().awaitRunning() sparkHttpServiceServer = Some(httpServer) } } override def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd): Unit = { - sparkHttpServiceServer.foreach(_.stopAndWait()) + sparkHttpServiceServer.foreach(_.stopAsync().awaitTerminated()) applicationEndLatch.countDown } @@ -183,7 +183,7 @@ abstract class AbstractSparkExecutionContext(sparkClassLoader: SparkClassLoader, SparkRuntimeEnv.stop().foreach(sc => applicationEndLatch.await()) } finally { try { - sparkDriveHttpService.stopAndWait() + sparkDriveHttpService.stopAsync().awaitTerminated() } finally { eventLoggerCloseable.close() } diff --git a/cdap-spark-core-base/src/test/java/io/cdap/cdap/app/runtime/spark/NoSparkClassLoaderTestRunner.java b/cdap-spark-core-base/src/test/java/io/cdap/cdap/app/runtime/spark/NoSparkClassLoaderTestRunner.java index 405e179f91dc..3b5c58237a90 100644 --- a/cdap-spark-core-base/src/test/java/io/cdap/cdap/app/runtime/spark/NoSparkClassLoaderTestRunner.java +++ b/cdap-spark-core-base/src/test/java/io/cdap/cdap/app/runtime/spark/NoSparkClassLoaderTestRunner.java @@ -49,6 +49,10 @@ private DelegateNoSparkClassLoader(ClassLoader delegate) { @Override public Class loadClass(String name) throws ClassNotFoundException { + Class loadedClass = findLoadedClass(name); + if (loadedClass != null) { + return loadedClass; + } if (name.startsWith(CDAP_PACKAGE_NAME)) { // Load all CDAP classes with this classloader. // We manually feed the original class into the delegated classloader to maintain the parent classloader diff --git a/cdap-spark-core-base/src/test/java/io/cdap/cdap/app/runtime/spark/SparkCredentialsUpdaterTest.java b/cdap-spark-core-base/src/test/java/io/cdap/cdap/app/runtime/spark/SparkCredentialsUpdaterTest.java index b5cfc5ad8cce..7532161f5009 100644 --- a/cdap-spark-core-base/src/test/java/io/cdap/cdap/app/runtime/spark/SparkCredentialsUpdaterTest.java +++ b/cdap-spark-core-base/src/test/java/io/cdap/cdap/app/runtime/spark/SparkCredentialsUpdaterTest.java @@ -70,7 +70,7 @@ long getNextUpdateDelay(Credentials credentials) throws IOException { UserGroupInformation.getCurrentUser().addToken(new Token<>(Bytes.toBytes("id"), Bytes.toBytes("pass"), new Text("kind"), new Text("service"))); - updater.startAndWait(); + updater.startAsync().awaitRunning(); try { List expectedFiles = new ArrayList<>(); expectedFiles.add(credentialsDir.append("credentials-1")); @@ -96,7 +96,7 @@ long getNextUpdateDelay(Credentials credentials) throws IOException { expectedFiles.add(credentialsDir.append("credentials-" + (i + 1))); } } finally { - updater.stopAndWait(); + updater.stopAsync().awaitTerminated(); } } @@ -115,7 +115,7 @@ long getNextUpdateDelay(Credentials credentials) throws IOException { } }; - updater.startAndWait(); + updater.startAsync().awaitRunning(); try { // Expect this loop to finish in 3 seconds because we don't want sleep for too long for testing cleanup for (int i = 1; i <= 5; i++) { @@ -130,7 +130,7 @@ long getNextUpdateDelay(Credentials credentials) throws IOException { updater.run(); Assert.assertEquals(3, credentialsDir.list().size()); } finally { - updater.stopAndWait(); + updater.stopAsync().awaitTerminated(); } } diff --git a/cdap-spark-core-base/src/test/java/io/cdap/cdap/app/runtime/spark/SparkTransactionHandlerTest.java b/cdap-spark-core-base/src/test/java/io/cdap/cdap/app/runtime/spark/SparkTransactionHandlerTest.java index 54cdd61636a6..643adab1e677 100644 --- a/cdap-spark-core-base/src/test/java/io/cdap/cdap/app/runtime/spark/SparkTransactionHandlerTest.java +++ b/cdap-spark-core-base/src/test/java/io/cdap/cdap/app/runtime/spark/SparkTransactionHandlerTest.java @@ -67,22 +67,22 @@ public class SparkTransactionHandlerTest { @BeforeClass public static void init() throws UnknownHostException { txManager = new TransactionManager(new Configuration()); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); txClient = new InMemoryTxSystemClient(txManager); sparkTxHandler = new SparkTransactionHandler(txClient); httpService = new SparkDriverHttpService("test", InetAddress.getLoopbackAddress().getCanonicalHostName(), sparkTxHandler); - httpService.startAndWait(); + httpService.startAsync().awaitRunning(); sparkTxClient = new SparkTransactionClient(httpService.getBaseURI()); } @AfterClass public static void finish() { - httpService.stopAndWait(); - txManager.stopAndWait(); + httpService.stopAsync().awaitTerminated(); + txManager.stopAsync().awaitTerminated(); } /** @@ -176,12 +176,12 @@ public Transaction startLong() { throw new IllegalStateException("Cannot start long transaction"); } }; - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); try { SparkTransactionHandler txHandler = new SparkTransactionHandler(new InMemoryTxSystemClient(txManager)); SparkDriverHttpService httpService = new SparkDriverHttpService( "test", InetAddress.getLoopbackAddress().getCanonicalHostName(), txHandler); - httpService.startAndWait(); + httpService.startAsync().awaitRunning(); try { // Start a job txHandler.jobStarted(1, ImmutableSet.of(2)); @@ -197,10 +197,10 @@ public Transaction startLong() { // End the job txHandler.jobEnded(1, false); } finally { - httpService.stopAndWait(); + httpService.stopAsync().awaitTerminated(); } } finally { - txManager.stopAndWait(); + txManager.stopAsync().awaitTerminated(); } } diff --git a/cdap-spark-core-base/src/test/java/io/cdap/cdap/app/runtime/spark/classloader/SparkRunnerClassLoaderTest.java b/cdap-spark-core-base/src/test/java/io/cdap/cdap/app/runtime/spark/classloader/SparkRunnerClassLoaderTest.java index 5fd0bb2f4c01..10c6ea6d10b1 100644 --- a/cdap-spark-core-base/src/test/java/io/cdap/cdap/app/runtime/spark/classloader/SparkRunnerClassLoaderTest.java +++ b/cdap-spark-core-base/src/test/java/io/cdap/cdap/app/runtime/spark/classloader/SparkRunnerClassLoaderTest.java @@ -16,7 +16,7 @@ package io.cdap.cdap.app.runtime.spark.classloader; -import com.google.common.io.Closeables; + import io.cdap.cdap.common.lang.ClassLoaders; import org.junit.Assert; import org.junit.Test; @@ -69,7 +69,11 @@ public void run() { } catch (Exception e) { exception.set(e); } finally { - Closeables.closeQuietly(secondCL); + try { + secondCL.close(); + } catch (Exception e) { + // ignore + } } } }; @@ -140,7 +144,11 @@ public void testCloseResourceStream() throws IOException { // After closing the ClassLoader, // reading from stream acquired through getResourceAsStream() should fail with an exception. - Closeables.closeQuietly(cl); + try { + cl.close(); + } catch (Exception e) { + // ignore + } is.read(); } } diff --git a/cdap-spark-core-base/src/test/java/io/cdap/cdap/app/runtime/spark/distributed/SparkExecutionServiceTest.java b/cdap-spark-core-base/src/test/java/io/cdap/cdap/app/runtime/spark/distributed/SparkExecutionServiceTest.java index 8138f4284e82..84a2226a3e9b 100644 --- a/cdap-spark-core-base/src/test/java/io/cdap/cdap/app/runtime/spark/distributed/SparkExecutionServiceTest.java +++ b/cdap-spark-core-base/src/test/java/io/cdap/cdap/app/runtime/spark/distributed/SparkExecutionServiceTest.java @@ -86,7 +86,7 @@ public void testCompletion() throws Exception { SparkExecutionService service = new SparkExecutionService(locationFactory, InetAddress.getLoopbackAddress().getCanonicalHostName(), programRunId, null); - service.startAndWait(); + service.startAsync().awaitRunning(); try { SparkExecutionClient client = new SparkExecutionClient(service.getBaseURI(), programRunId); @@ -99,7 +99,7 @@ public void testCompletion() throws Exception { // Call complete to notify the service it has been stopped client.completed(null); } finally { - service.stopAndWait(); + service.stopAsync().awaitTerminated(); } } @@ -111,7 +111,7 @@ public void testExplicitStop() throws Exception { SparkExecutionService service = new SparkExecutionService(locationFactory, InetAddress.getLoopbackAddress().getCanonicalHostName(), programRunId, null); - service.startAndWait(); + service.startAsync().awaitRunning(); try { final SparkExecutionClient client = new SparkExecutionClient(service.getBaseURI(), programRunId); @@ -122,7 +122,19 @@ public void testExplicitStop() throws Exception { } // Stop the program from the service side - ListenableFuture stopFuture = service.stop(); + com.google.common.util.concurrent.SettableFuture stopFuture = + com.google.common.util.concurrent.SettableFuture.create(); + service.addListener(new Service.Listener() { + @Override + public void terminated(Service.State from) { + stopFuture.set(Service.State.TERMINATED); + } + @Override + public void failed(Service.State from, Throwable failure) { + stopFuture.setException(failure); + } + }, com.google.common.util.concurrent.MoreExecutors.directExecutor()); + service.stopAsync(); // Expect some future heartbeats will receive the STOP command Tasks.waitFor(true, new Callable() { @@ -139,7 +151,7 @@ public Boolean call() throws Exception { // The stop future of the service should be completed after the client.completed call. stopFuture.get(5, TimeUnit.SECONDS); } finally { - service.stopAndWait(); + service.stopAsync().awaitTerminated(); } } @@ -153,7 +165,7 @@ public void testWorkflowToken() throws Exception { SparkExecutionService service = new SparkExecutionService(locationFactory, InetAddress.getLoopbackAddress().getCanonicalHostName(), programRunId, token); - service.startAndWait(); + service.startAsync().awaitRunning(); try { SparkExecutionClient client = new SparkExecutionClient(service.getBaseURI(), programRunId); @@ -172,7 +184,7 @@ public void testWorkflowToken() throws Exception { clientToken.put("completed", "true"); client.completed(clientToken); } finally { - service.stopAndWait(); + service.stopAsync().awaitTerminated(); } // The token on the service side should get updated after the completed call. @@ -199,7 +211,7 @@ public void testWriteCredentials() throws Exception { SparkExecutionService service = new SparkExecutionService(locationFactory, InetAddress.getLoopbackAddress().getCanonicalHostName(), programRunId, null); - service.startAndWait(); + service.startAsync().awaitRunning(); try { SparkExecutionClient client = new SparkExecutionClient(service.getBaseURI(), programRunId); @@ -227,7 +239,7 @@ public void testWriteCredentials() throws Exception { // Call complete to notify the service it has been stopped client.completed(null); } finally { - service.stopAndWait(); + service.stopAsync().awaitTerminated(); } } } diff --git a/cdap-spark-core3_2.12/pom.xml b/cdap-spark-core3_2.12/pom.xml index 586a6c326f29..fe9465efbf94 100644 --- a/cdap-spark-core3_2.12/pom.xml +++ b/cdap-spark-core3_2.12/pom.xml @@ -211,6 +211,15 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + net.alchim31.maven scala-maven-plugin @@ -353,7 +362,6 @@ org.apache.maven.plugins maven-jar-plugin - 2.4 jar @@ -361,6 +369,7 @@ ${project.build.directory}/libexec ${project.groupId}.${project.build.finalName} + false jar @@ -372,6 +381,7 @@ ${project.build.directory}/k8s ${project.groupId}.${project.build.finalName} + false jar @@ -443,7 +453,7 @@ org.apache.maven.plugins maven-shade-plugin - 3.2.1 + 3.5.2 prepare-package diff --git a/cdap-spark-python/pom.xml b/cdap-spark-python/pom.xml index 634af040ea06..c30316a4ae46 100644 --- a/cdap-spark-python/pom.xml +++ b/cdap-spark-python/pom.xml @@ -36,4 +36,18 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + diff --git a/cdap-standalone/pom.xml b/cdap-standalone/pom.xml index 1a2dce98b489..30ac6b56c29d 100644 --- a/cdap-standalone/pom.xml +++ b/cdap-standalone/pom.xml @@ -364,10 +364,18 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 17 + 17 + + org.apache.maven.plugins maven-jar-plugin - 2.4 *.xml @@ -471,7 +479,6 @@ org.apache.maven.plugins maven-jar-plugin - 2.4 org.apache.maven.plugins diff --git a/cdap-standalone/src/main/java/io/cdap/cdap/StandaloneMain.java b/cdap-standalone/src/main/java/io/cdap/cdap/StandaloneMain.java index e2a58907d40c..4df66d5b60d0 100644 --- a/cdap-standalone/src/main/java/io/cdap/cdap/StandaloneMain.java +++ b/cdap-standalone/src/main/java/io/cdap/cdap/StandaloneMain.java @@ -248,25 +248,25 @@ public void startUp() throws Exception { ConfigurationLogger.logImportantConfig(cConf); if (messagingService instanceof Service) { - ((Service) messagingService).startAndWait(); + ((Service) messagingService).startAsync().awaitRunning(); } // TODO: CDAP-7688, remove next line after the issue is resolved - injector.getInstance(MessagingHttpService.class).startAndWait(); + injector.getInstance(MessagingHttpService.class).startAsync().awaitRunning(); if (txService != null) { - txService.startAndWait(); + txService.startAsync().awaitRunning(); } // Define all StructuredTable before starting any services that need StructuredTable StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class)); metadataStorage.createIndex(); - metricsCollectionService.startAndWait(); - datasetOpExecutorService.startAndWait(); - datasetService.startAndWait(); - serviceStore.startAndWait(); + metricsCollectionService.startAsync().awaitRunning(); + datasetOpExecutorService.startAsync().awaitRunning(); + datasetService.startAsync().awaitRunning(); + serviceStore.startAsync().awaitRunning(); remoteExecutionTwillRunnerService.start(); - metadataSubscriberService.startAndWait(); + metadataSubscriberService.startAsync().awaitRunning(); // Validate the logging pipeline configuration. // Do it explicitly as Standalone doesn't have a separate master check phase as the distributed does. @@ -275,18 +275,20 @@ public void startUp() throws Exception { // since log appender instantiates a dataset. logAppenderInitializer.initialize(); - runtimeServer.startAndWait(); - Service.State state = appFabricServer.startAndWait(); + runtimeServer.startAsync().awaitRunning(); + appFabricServer.startAsync().awaitRunning(); + Service.State state = appFabricServer.state(); if (state != Service.State.RUNNING) { throw new Exception("Failed to start Application Fabric"); } - state = appFabricProcessorService.startAndWait(); + appFabricProcessorService.startAsync().awaitRunning(); + state = appFabricProcessorService.state(); if (state != Service.State.RUNNING) { throw new Exception("Failed to start Application Fabric Processor"); } - artifactLocalizerService.startAndWait(); + artifactLocalizerService.startAsync().awaitRunning(); // NOTE: As the artifact localizer client does not use service discovery for port discovery, // We need to set the port after starting the localizer service. cConf.setInt(Constants.ArtifactLocalizer.PORT, artifactLocalizerService.getPort()); @@ -294,27 +296,27 @@ public void startUp() throws Exception { injector.getInstance( Key.get(CConfiguration.class, Names.named(PreviewConfigModule.PREVIEW_CCONF))) .setInt(Constants.ArtifactLocalizer.PORT, artifactLocalizerService.getPort()); - previewHttpServer.startAndWait(); - previewRunnerManager.startAndWait(); + previewHttpServer.startAsync().awaitRunning(); + previewRunnerManager.startAsync().awaitRunning(); - metricsQueryService.startAndWait(); - logQueryService.startAndWait(); - router.startAndWait(); + metricsQueryService.startAsync().awaitRunning(); + logQueryService.startAsync().awaitRunning(); + router.startAsync().awaitRunning(); if (userInterfaceService != null) { - userInterfaceService.startAndWait(); + userInterfaceService.startAsync().awaitRunning(); } if (SecurityUtil.isManagedSecurity(cConf)) { - externalAuthenticationServer.startAndWait(); + externalAuthenticationServer.startAsync().awaitRunning(); } - metadataService.startAndWait(); - operationalStatsService.startAndWait(); - secureStoreService.startAndWait(); - supportBundleInternalService.startAndWait(); - eventPublishManager.startAndWait(); - eventSubscriberManager.startAndWait(); + metadataService.startAsync().awaitRunning(); + operationalStatsService.startAsync().awaitRunning(); + secureStoreService.startAsync().awaitRunning(); + supportBundleInternalService.startAsync().awaitRunning(); + eventPublishManager.startAsync().awaitRunning(); + eventSubscriberManager.startAsync().awaitRunning(); String protocol = sslEnabled ? "https" : "http"; int dashboardPort = sslEnabled @@ -334,52 +336,52 @@ public void shutDown() { try { // order matters: first shut down UI 'cause it will stop working after router is down if (userInterfaceService != null) { - userInterfaceService.stopAndWait(); + userInterfaceService.stopAsync().awaitTerminated(); } // shut down router to stop all incoming traffic - router.stopAndWait(); + router.stopAsync().awaitTerminated(); - secureStoreService.stopAndWait(); - supportBundleInternalService.stopAndWait(); - operationalStatsService.stopAndWait(); + secureStoreService.stopAsync().awaitTerminated(); + supportBundleInternalService.stopAsync().awaitTerminated(); + operationalStatsService.stopAsync().awaitTerminated(); // Stop all services that requires tx service - metadataSubscriberService.stopAndWait(); - metadataService.stopAndWait(); + metadataSubscriberService.stopAsync().awaitTerminated(); + metadataService.stopAsync().awaitTerminated(); remoteExecutionTwillRunnerService.stop(); - serviceStore.stopAndWait(); - previewRunnerManager.stopAndWait(); - previewHttpServer.stopAndWait(); - artifactLocalizerService.stopAndWait(); - eventPublishManager.stopAndWait(); - eventSubscriberManager.stopAndWait(); + serviceStore.stopAsync().awaitTerminated(); + previewRunnerManager.stopAsync().awaitTerminated(); + previewHttpServer.stopAsync().awaitTerminated(); + artifactLocalizerService.stopAsync().awaitTerminated(); + eventPublishManager.stopAsync().awaitTerminated(); + eventSubscriberManager.stopAsync().awaitTerminated(); // app fabric will also stop all programs - appFabricServer.stopAndWait(); - appFabricProcessorService.stopAndWait(); - runtimeServer.stopAndWait(); + appFabricServer.stopAsync().awaitTerminated(); + appFabricProcessorService.stopAsync().awaitTerminated(); + runtimeServer.stopAsync().awaitTerminated(); // all programs are stopped: dataset service, metrics, transactions can stop now - datasetService.stopAndWait(); - datasetOpExecutorService.stopAndWait(); + datasetService.stopAsync().awaitTerminated(); + datasetOpExecutorService.stopAsync().awaitTerminated(); - logQueryService.stopAndWait(); + logQueryService.stopAsync().awaitTerminated(); - metricsCollectionService.stopAndWait(); - metricsQueryService.stopAndWait(); + metricsCollectionService.stopAsync().awaitTerminated(); + metricsQueryService.stopAsync().awaitTerminated(); if (txService != null) { - txService.stopAndWait(); + txService.stopAsync().awaitTerminated(); } if (SecurityUtil.isManagedSecurity(cConf)) { // auth service is on the side anyway - externalAuthenticationServer.stopAndWait(); + externalAuthenticationServer.stopAsync().awaitTerminated(); } // TODO: CDAP-7688, remove next line after the issue is resolved - injector.getInstance(MessagingHttpService.class).startAndWait(); + injector.getInstance(MessagingHttpService.class).startAsync().awaitRunning(); if (messagingService instanceof Service) { - ((Service) messagingService).stopAndWait(); + ((Service) messagingService).stopAsync().awaitTerminated(); } logAppenderInitializer.close(); diff --git a/cdap-standalone/src/main/resources/cdap-env.sh b/cdap-standalone/src/main/resources/cdap-env.sh index 89ad34dcf5bb..50e6edf5eb53 100644 --- a/cdap-standalone/src/main/resources/cdap-env.sh +++ b/cdap-standalone/src/main/resources/cdap-env.sh @@ -17,3 +17,6 @@ export KILL_ON_OOM_OPTS="-XX:OnOutOfMemoryError=\"kill -9 %p\"" # Uncomment to perform Java heap dump on OutOfMemory errors # export HEAPDUMP_ON_OOM=true + +# Java 17 support for Wrangler system app and Apache Spark local execution deployment +export JAVA_OPTS="${JAVA_OPTS} --add-opens java.base/java.time=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.invoke=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/sun.nio.ch=ALL-UNNAMED --add-opens java.security/java.security=ALL-UNNAMED" diff --git a/cdap-standalone/src/main/resources/cdap-site.xml b/cdap-standalone/src/main/resources/cdap-site.xml index 337d66e0adda..761fcf731a4b 100644 --- a/cdap-standalone/src/main/resources/cdap-site.xml +++ b/cdap-standalone/src/main/resources/cdap-site.xml @@ -316,4 +316,14 @@ + + + app.program.jvm.opts + -XX:+IgnoreUnrecognizedVMOptions --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.invoke=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/sun.nio.ch=ALL-UNNAMED --add-opens java.security/java.security=ALL-UNNAMED + + JVM options used by spawned program execution containers. + These encapsulation bypass open flags are mandatory to execute Spark 3.x workflows under Java 17. + + + diff --git a/cdap-standalone/src/test/java/io/cdap/cdap/StandaloneMainTest.java b/cdap-standalone/src/test/java/io/cdap/cdap/StandaloneMainTest.java index 320595365b32..651bffca67e4 100644 --- a/cdap-standalone/src/test/java/io/cdap/cdap/StandaloneMainTest.java +++ b/cdap-standalone/src/test/java/io/cdap/cdap/StandaloneMainTest.java @@ -33,6 +33,14 @@ public class StandaloneMainTest { @Test public void testInjector() { + try { + System.out.println("DEBUG: Service$Listener source: " + + com.google.common.util.concurrent.Service.Listener.class.getProtectionDomain().getCodeSource().getLocation()); + System.out.println("DEBUG: ClassReader source: " + + org.objectweb.asm.ClassReader.class.getProtectionDomain().getCodeSource().getLocation()); + } catch (Exception e) { + e.printStackTrace(); + } StandaloneMain sdk = StandaloneMain.create(CConfiguration.create(), new Configuration()); PreviewHttpServer previewHttpServer = sdk.getInjector().getInstance(PreviewHttpServer.class); PreviewRunnerManager previewRunnerManager = sdk.getInjector().getInstance(PreviewRunnerManager.class); @@ -40,10 +48,11 @@ public void testInjector() { Assert.assertSame(previewRunnerManager, previewRunStopper); TransactionManager txManager = sdk.getInjector().getInstance(TransactionManager.class); - txManager.startAndWait(); - previewHttpServer.startAndWait(); - ((Service) previewRunnerManager).startAndWait(); - ((Service) previewRunnerManager).stopAndWait(); - previewHttpServer.stopAndWait(); + txManager.startAsync().awaitRunning(); + previewHttpServer.startAsync().awaitRunning(); + ((Service) previewRunnerManager).startAsync().awaitRunning(); + ((Service) previewRunnerManager).stopAsync().awaitTerminated(); + previewHttpServer.stopAsync().awaitTerminated(); + txManager.stopAsync().awaitTerminated(); } } diff --git a/cdap-storage-ext-spanner/pom.xml b/cdap-storage-ext-spanner/pom.xml index c805ff445b49..99175e165b01 100644 --- a/cdap-storage-ext-spanner/pom.xml +++ b/cdap-storage-ext-spanner/pom.xml @@ -89,6 +89,20 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + dist @@ -120,7 +134,6 @@ org.apache.maven.plugins maven-jar-plugin - 2.4 jar @@ -128,6 +141,7 @@ ${project.build.directory}/libexec ${project.groupId}.${project.build.finalName} + false jar diff --git a/cdap-storage-spi/pom.xml b/cdap-storage-spi/pom.xml index 526029644b34..550b831014ae 100644 --- a/cdap-storage-spi/pom.xml +++ b/cdap-storage-spi/pom.xml @@ -48,10 +48,18 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + org.apache.maven.plugins maven-jar-plugin - 2.4 test-jar diff --git a/cdap-support-bundle/pom.xml b/cdap-support-bundle/pom.xml index 7a5ad40437c5..fef64084c60f 100644 --- a/cdap-support-bundle/pom.xml +++ b/cdap-support-bundle/pom.xml @@ -79,4 +79,18 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + diff --git a/cdap-support-bundle/src/test/java/io/cdap/cdap/SupportBundleTestBase.java b/cdap-support-bundle/src/test/java/io/cdap/cdap/SupportBundleTestBase.java index 5e9d40f882eb..c21fc6236a11 100644 --- a/cdap-support-bundle/src/test/java/io/cdap/cdap/SupportBundleTestBase.java +++ b/cdap-support-bundle/src/test/java/io/cdap/cdap/SupportBundleTestBase.java @@ -17,7 +17,7 @@ package io.cdap.cdap; import com.google.common.base.Preconditions; -import com.google.common.io.Closeables; + import com.google.common.util.concurrent.Service; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -186,40 +186,40 @@ protected static void initializeAndStartServices(CConfiguration cConf, Module ov messagingService = injector.getInstance(MessagingService.class); if (messagingService instanceof Service) { - ((Service) messagingService).startAndWait(); + ((Service) messagingService).startAsync().awaitRunning(); } txManager = injector.getInstance(TransactionManager.class); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); // Define all StructuredTable before starting any services that need StructuredTable StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class)); metadataStorage = injector.getInstance(MetadataStorage.class); metadataStorage.createIndex(); dsOpService = injector.getInstance(DatasetOpExecutorService.class); - dsOpService.startAndWait(); + dsOpService.startAsync().awaitRunning(); datasetService = injector.getInstance(DatasetService.class); - datasetService.startAndWait(); + datasetService.startAsync().awaitRunning(); appFabricServer = injector.getInstance(AppFabricServer.class); - appFabricServer.startAndWait(); + appFabricServer.startAsync().awaitRunning(); appFabricProcessor = injector.getInstance(AppFabricProcessorService.class); - appFabricProcessor.startAndWait(); + appFabricProcessor.startAsync().awaitRunning(); DiscoveryServiceClient discoveryClient = injector.getInstance(DiscoveryServiceClient.class); appFabricEndpointStrategy = new RandomEndpointStrategy( () -> discoveryClient.discover(Constants.Service.APP_FABRIC_HTTP)); metricsCollectionService = injector.getInstance(MetricsCollectionService.class); - metricsCollectionService.startAndWait(); + metricsCollectionService.startAsync().awaitRunning(); serviceStore = injector.getInstance(ServiceStore.class); - serviceStore.startAndWait(); + serviceStore.startAsync().awaitRunning(); metadataService = injector.getInstance(MetadataService.class); - metadataService.startAndWait(); + metadataService.startAsync().awaitRunning(); metadataSubscriberService = injector.getInstance(MetadataSubscriberService.class); - metadataSubscriberService.startAndWait(); + metadataSubscriberService.startAsync().awaitRunning(); logQueryService = injector.getInstance(LogQueryService.class); - logQueryService.startAndWait(); + logQueryService.startAsync().awaitRunning(); locationFactory = getInjector().getInstance(LocationFactory.class); supportBundleInternalService = injector.getInstance(SupportBundleInternalService.class); - supportBundleInternalService.startAndWait(); + supportBundleInternalService.startAsync().awaitRunning(); Scheduler programScheduler = injector.getInstance(Scheduler.class); // Wait for the scheduler to be functional. @@ -234,21 +234,27 @@ protected static void initializeAndStartServices(CConfiguration cConf, Module ov @AfterClass public static void afterClass() throws IOException { - appFabricServer.stopAndWait(); - appFabricProcessor.stopAndWait(); - metricsCollectionService.stopAndWait(); - datasetService.stopAndWait(); - dsOpService.stopAndWait(); - txManager.stopAndWait(); - serviceStore.stopAndWait(); - metadataSubscriberService.stopAndWait(); - metadataService.stopAndWait(); - logQueryService.stopAndWait(); + appFabricServer.stopAsync().awaitTerminated(); + appFabricProcessor.stopAsync().awaitTerminated(); + metricsCollectionService.stopAsync().awaitTerminated(); + datasetService.stopAsync().awaitTerminated(); + dsOpService.stopAsync().awaitTerminated(); + txManager.stopAsync().awaitTerminated(); + serviceStore.stopAsync().awaitTerminated(); + metadataSubscriberService.stopAsync().awaitTerminated(); + metadataService.stopAsync().awaitTerminated(); + logQueryService.stopAsync().awaitTerminated(); if (messagingService instanceof Service) { - ((Service) messagingService).stopAndWait(); + ((Service) messagingService).stopAsync().awaitTerminated(); + } + try { + if (metadataStorage != null) { + metadataStorage.close(); + } + } catch (Exception e) { + // Ignore } - Closeables.closeQuietly(metadataStorage); - supportBundleInternalService.stopAndWait(); + supportBundleInternalService.stopAsync().awaitTerminated(); } protected static CConfiguration createBasicCconf() throws IOException { diff --git a/cdap-system-app-api/src/main/java/io/cdap/cdap/api/service/SystemNamespaceAdmin.java b/cdap-system-app-api/src/main/java/io/cdap/cdap/api/service/SystemNamespaceAdmin.java index 7c731d9cfe9e..066d455d7ef4 100644 --- a/cdap-system-app-api/src/main/java/io/cdap/cdap/api/service/SystemNamespaceAdmin.java +++ b/cdap-system-app-api/src/main/java/io/cdap/cdap/api/service/SystemNamespaceAdmin.java @@ -21,12 +21,12 @@ import java.util.List; /** - * Interface for list all namespaces for system service + * Interface for list all namespaces for system service. */ public interface SystemNamespaceAdmin { /** - * List all the namespaces + * List all the namespaces. */ List listNamespaces() throws Exception; } diff --git a/cdap-system-app-api/src/main/java/io/cdap/cdap/api/service/http/SystemHttpServiceContext.java b/cdap-system-app-api/src/main/java/io/cdap/cdap/api/service/http/SystemHttpServiceContext.java index 51f3eef489be..c7a515b97d1b 100644 --- a/cdap-system-app-api/src/main/java/io/cdap/cdap/api/service/http/SystemHttpServiceContext.java +++ b/cdap-system-app-api/src/main/java/io/cdap/cdap/api/service/http/SystemHttpServiceContext.java @@ -69,12 +69,12 @@ Map evaluateMacros(String namespace, Map propert /** * Get preferences for the given namespace. - *

- * This method fetches preferences for the supplied namespace when the method is unvoked, unlike + * + *

This method fetches preferences for the supplied namespace when the method is unvoked, unlike * {@link io.cdap.cdap.api.RuntimeContext#getRuntimeArguments()} which returns arguments at the * time the context was created. - *

- * This might be a network call, depending on the underlying implementation. + * + *

This might be a network call, depending on the underlying implementation. * * @param namespace the name of the namespace to fetch preferences for. * @param resolved true if resolved properties are desired. @@ -88,12 +88,14 @@ default Map getPreferencesForNamespace(String namespace, boolean } /** + * Gets the context access enforcer. + * * @return {@link ContextAccessEnforcer} that can be used to enforce access for current request */ ContextAccessEnforcer getContextAccessEnforcer(); /** - * Runs the task from {@link RunnableTaskRequest} remotely on a task worker + * Runs the task from {@link RunnableTaskRequest} remotely on a task worker. * * @param runnableTaskRequest Details of the task * @return byte[] result @@ -102,7 +104,7 @@ default Map getPreferencesForNamespace(String namespace, boolean byte[] runTask(RunnableTaskRequest runnableTaskRequest) throws Exception; /** - * Returns boolean indicating whether remote task execution is enabled + * Returns boolean indicating whether remote task execution is enabled. */ boolean isRemoteTaskEnabled(); } diff --git a/cdap-system-app-unit-test/pom.xml b/cdap-system-app-unit-test/pom.xml index 6208d6cbab5e..da3c2355b035 100644 --- a/cdap-system-app-unit-test/pom.xml +++ b/cdap-system-app-unit-test/pom.xml @@ -49,4 +49,18 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + diff --git a/cdap-test/pom.xml b/cdap-test/pom.xml index 344cde3a8fc0..547d57fef53c 100644 --- a/cdap-test/pom.xml +++ b/cdap-test/pom.xml @@ -45,4 +45,18 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + diff --git a/cdap-test/src/main/java/io/cdap/cdap/test/MetricsManager.java b/cdap-test/src/main/java/io/cdap/cdap/test/MetricsManager.java index 82dc6b5e780b..45870eec0586 100644 --- a/cdap-test/src/main/java/io/cdap/cdap/test/MetricsManager.java +++ b/cdap-test/src/main/java/io/cdap/cdap/test/MetricsManager.java @@ -134,8 +134,8 @@ public void waitForTotalMetricCount(Map tags, String metricName, // Min sleep time is 10ms, max sleep time is 1 seconds long sleepMillis = Math.max(10, Math.min(timeoutUnit.toMillis(timeout) / 10, TimeUnit.SECONDS.toMillis(1))); - Stopwatch stopwatch = new Stopwatch().start(); - while (value < count && stopwatch.elapsedTime(timeoutUnit) < timeout) { + Stopwatch stopwatch = Stopwatch.createStarted(); + while (value < count && stopwatch.elapsed(timeoutUnit) < timeout) { TimeUnit.MILLISECONDS.sleep(sleepMillis); value = getTotalMetric(tags, metricName); } diff --git a/cdap-tms-tests/pom.xml b/cdap-tms-tests/pom.xml index e0c5c292abf3..e5f185876a89 100644 --- a/cdap-tms-tests/pom.xml +++ b/cdap-tms-tests/pom.xml @@ -123,6 +123,15 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + org.sonatype.central diff --git a/cdap-tms/pom.xml b/cdap-tms/pom.xml index 3f845586dac2..d25178d42334 100644 --- a/cdap-tms/pom.xml +++ b/cdap-tms/pom.xml @@ -151,10 +151,18 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 17 + 17 + + org.apache.maven.plugins maven-jar-plugin - 2.4 test-jar diff --git a/cdap-tms/src/main/java/io/cdap/cdap/messaging/distributed/LeaderElectionMessagingService.java b/cdap-tms/src/main/java/io/cdap/cdap/messaging/distributed/LeaderElectionMessagingService.java index 6e1a8b8f84e2..0e6ff7ffedd0 100644 --- a/cdap-tms/src/main/java/io/cdap/cdap/messaging/distributed/LeaderElectionMessagingService.java +++ b/cdap-tms/src/main/java/io/cdap/cdap/messaging/distributed/LeaderElectionMessagingService.java @@ -127,14 +127,14 @@ public void follower() { latch.countDown(); } }); - leaderElection.startAndWait(); + leaderElection.startAsync().awaitRunning(); latch.await(); } @Override protected void shutDown() throws Exception { try { - leaderElection.stopAndWait(); + leaderElection.stopAsync().awaitTerminated(); } catch (Exception e) { // It can happen if it is currently disconnected from ZK. There is no harm in just continue the shutdown process. LOG.warn("Exception during shutting down leader election", e); @@ -209,7 +209,7 @@ private void updateDelegate(@Nullable DelegateService newService) { } if (oldService != null) { - oldService.stopAndWait(); + oldService.stopAsync().awaitTerminated(); } } @@ -217,11 +217,11 @@ private void fencingStart(final DelegateService service) { Runnable runnable = new Runnable() { @Override public void run() { - service.startAndWait(); + service.startAsync().awaitRunning(); // If failed to mark the service to become available, this means the follower() call happened before this, // so just go ahead and shutdown the service. if (!delegate.attemptMark(service, true)) { - service.stopAndWait(); + service.stopAsync().awaitTerminated(); } } }; @@ -260,15 +260,15 @@ private final class DelegateService extends AbstractIdleService { @Override protected void startUp() throws Exception { - messagingService.startAndWait(); - httpService.startAndWait(); + messagingService.startAsync().awaitRunning(); + httpService.startAsync().awaitRunning(); } @Override protected void shutDown() throws Exception { try { - httpService.stopAndWait(); - messagingService.stopAndWait(); + httpService.stopAsync().awaitTerminated(); + messagingService.stopAsync().awaitTerminated(); } finally { // Clear the table cache on shutting down. cacheProvider.clear(); diff --git a/cdap-tms/src/main/java/io/cdap/cdap/messaging/server/MessagingHttpService.java b/cdap-tms/src/main/java/io/cdap/cdap/messaging/server/MessagingHttpService.java index d828231be269..0ccc63004ac1 100644 --- a/cdap-tms/src/main/java/io/cdap/cdap/messaging/server/MessagingHttpService.java +++ b/cdap-tms/src/main/java/io/cdap/cdap/messaging/server/MessagingHttpService.java @@ -16,7 +16,7 @@ package io.cdap.cdap.messaging.server; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.util.concurrent.AbstractIdleService; import com.google.inject.Inject; import com.google.inject.name.Named; @@ -90,7 +90,7 @@ public void handle(Throwable t, HttpRequest request, HttpResponder responder) { private void logWithTrace(HttpRequest request, Throwable t) { LOG.trace("Error in handling request={} {} for user={}:", request.method().name(), request.uri(), - Objects.firstNonNull(SecurityRequestContext.getUserId(), ""), t); + MoreObjects.firstNonNull(SecurityRequestContext.getUserId(), ""), t); } }) .setHttpHandlers(handlers); diff --git a/cdap-tms/src/main/java/io/cdap/cdap/messaging/service/CoreMessagingService.java b/cdap-tms/src/main/java/io/cdap/cdap/messaging/service/CoreMessagingService.java index 16fa1735ed82..424a5f8cd9f5 100644 --- a/cdap-tms/src/main/java/io/cdap/cdap/messaging/service/CoreMessagingService.java +++ b/cdap-tms/src/main/java/io/cdap/cdap/messaging/service/CoreMessagingService.java @@ -17,7 +17,7 @@ package io.cdap.cdap.messaging.service; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.base.Throwables; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; @@ -195,7 +195,7 @@ public RollbackDetail publish(StoreRequest request) throws TopicNotFoundExceptio } return messageTableWriterCache.get(request.getTopicId()).persist(request, metadata); } catch (ExecutionException e) { - Throwable cause = Objects.firstNonNull(e.getCause(), e); + Throwable cause = MoreObjects.firstNonNull(e.getCause(), e); Throwables.propagateIfPossible(cause, TopicNotFoundException.class, IOException.class); throw Throwables.propagate(e); } @@ -207,7 +207,7 @@ public void storePayload(StoreRequest request) throws TopicNotFoundException, IO TopicMetadata metadata = topicCache.get(request.getTopicId()); payloadTableWriterCache.get(request.getTopicId()).persist(request, metadata); } catch (ExecutionException e) { - Throwable cause = Objects.firstNonNull(e.getCause(), e); + Throwable cause = MoreObjects.firstNonNull(e.getCause(), e); Throwables.propagateIfPossible(cause, TopicNotFoundException.class, IOException.class); throw Throwables.propagate(e); } @@ -267,7 +267,13 @@ private void createSystemTopic(TopicId topicId, Queue creationFailureTo protected void shutDown() throws Exception { messageTableWriterCache.invalidateAll(); payloadTableWriterCache.invalidateAll(); - Closeables.closeQuietly(tableFactory); + if (tableFactory instanceof AutoCloseable) { + try { + ((AutoCloseable) tableFactory).close(); + } catch (Exception e) { + // Ignore + } + } LOG.info("Core Messaging Service stopped"); } @@ -441,7 +447,7 @@ private TopicMetadata getTopic(TopicId topicId) throws TopicNotFoundException, I try { return topicCache.get(topicId); } catch (ExecutionException e) { - Throwable cause = Objects.firstNonNull(e.getCause(), e); + Throwable cause = MoreObjects.firstNonNull(e.getCause(), e); Throwables.propagateIfPossible(cause, TopicNotFoundException.class, IOException.class); throw Throwables.propagate(e.getCause()); } diff --git a/cdap-tms/src/main/java/io/cdap/cdap/messaging/store/leveldb/LevelDBTableFactory.java b/cdap-tms/src/main/java/io/cdap/cdap/messaging/store/leveldb/LevelDBTableFactory.java index f1729714327d..251a54b3c81e 100644 --- a/cdap-tms/src/main/java/io/cdap/cdap/messaging/store/leveldb/LevelDBTableFactory.java +++ b/cdap-tms/src/main/java/io/cdap/cdap/messaging/store/leveldb/LevelDBTableFactory.java @@ -195,12 +195,35 @@ public void close() { this.metadataTable = null; } if (metadataTable != null) { - Closeables.closeQuietly(metadataTable.getLevelDB()); + DB db = metadataTable.getLevelDB(); + if (db instanceof AutoCloseable) { + try { + ((AutoCloseable) db).close(); + } catch (Exception e) { + // Ignore + } + } } Collection dbs = levelDBs.values(); - dbs.forEach(Closeables::closeQuietly); + for (DB db : dbs) { + if (db instanceof AutoCloseable) { + try { + ((AutoCloseable) db).close(); + } catch (Exception e) { + // Ignore + } + } + } dbs.clear(); - partitionedLevelDBs.values().forEach(Closeables::closeQuietly); + for (LevelDBPartitionManager pm : partitionedLevelDBs.values()) { + if (pm instanceof AutoCloseable) { + try { + ((AutoCloseable) pm).close(); + } catch (Exception e) { + // Ignore + } + } + } partitionedLevelDBs.clear(); } @@ -312,7 +335,14 @@ public void run() { break; } // We can safely remove and close the levelDB as no one should be accessing them anymore - Closeables.closeQuietly(levelDBs.remove(dataDBPath)); + DB db = levelDBs.remove(dataDBPath); + if (db instanceof AutoCloseable) { + try { + ((AutoCloseable) db).close(); + } catch (Exception e) { + // Ignore + } + } filesToDelete.add(dataDBPath); // Payload table @@ -321,7 +351,14 @@ public void run() { break; } // We can safely remove and close the levelDB as no one should be accessing them anymore - Closeables.closeQuietly(levelDBs.remove(dataDBPath)); + db = levelDBs.remove(dataDBPath); + if (db instanceof AutoCloseable) { + try { + ((AutoCloseable) db).close(); + } catch (Exception e) { + // Ignore + } + } filesToDelete.add(dataDBPath); } diff --git a/cdap-tms/src/main/java/io/cdap/cdap/messaging/subscriber/AbstractMessagingSubscriberService.java b/cdap-tms/src/main/java/io/cdap/cdap/messaging/subscriber/AbstractMessagingSubscriberService.java index a1054eed6350..97e2b8daad67 100644 --- a/cdap-tms/src/main/java/io/cdap/cdap/messaging/subscriber/AbstractMessagingSubscriberService.java +++ b/cdap-tms/src/main/java/io/cdap/cdap/messaging/subscriber/AbstractMessagingSubscriberService.java @@ -185,7 +185,7 @@ protected String processMessages(Iterable> messages) th List> currentTxMessages; String lastMessageId = null; MessageTrackingIterator iterator = new MessageTrackingIterator(messages.iterator()); - Stopwatch stopwatch = new Stopwatch(); + Stopwatch stopwatch = Stopwatch.createUnstarted(); while (iterator.hasNext()) { currentTxMessages = new ArrayList<>(); diff --git a/cdap-tms/src/test/java/io/cdap/cdap/messaging/distributed/LeaderElectionMessagingServiceTest.java b/cdap-tms/src/test/java/io/cdap/cdap/messaging/distributed/LeaderElectionMessagingServiceTest.java index c1bd9925bf65..ba2ae0778378 100644 --- a/cdap-tms/src/test/java/io/cdap/cdap/messaging/distributed/LeaderElectionMessagingServiceTest.java +++ b/cdap-tms/src/test/java/io/cdap/cdap/messaging/distributed/LeaderElectionMessagingServiceTest.java @@ -118,11 +118,11 @@ public void testTransition() throws Throwable { // Start a messaging service, which would becomes leader ZKClientService zkClient1 = injector1.getInstance(ZKClientService.class); - zkClient1.startAndWait(); + zkClient1.startAsync().awaitRunning(); final MessagingService firstService = injector1.getInstance(MessagingService.class); if (firstService instanceof Service) { - ((Service) firstService).startAndWait(); + ((Service) firstService).startAsync().awaitRunning(); } // Publish a message with the leader @@ -130,11 +130,11 @@ public void testTransition() throws Throwable { // Start another messaging service, this one would be follower ZKClientService zkClient2 = injector2.getInstance(ZKClientService.class); - zkClient2.startAndWait(); + zkClient2.startAsync().awaitRunning(); final MessagingService secondService = injector2.getInstance(MessagingService.class); if (secondService instanceof Service) { - ((Service) secondService).startAndWait(); + ((Service) secondService).startAsync().awaitRunning(); } // Try to call the follower, should get service unavailable. @@ -175,7 +175,7 @@ public List call() throws Throwable { // Shutdown the current leader. The session timeout one should becomes leader again. if (secondService instanceof Service) { - ((Service) secondService).stopAndWait(); + ((Service) secondService).stopAsync().awaitTerminated(); } // Try to fetch message from the current leader again. @@ -201,8 +201,8 @@ public List call() throws Throwable { Assert.assertEquals(Arrays.asList("Testing1", "Testing2"), messages); - zkClient1.stopAndWait(); - zkClient2.stopAndWait(); + zkClient1.stopAsync().awaitTerminated(); + zkClient2.stopAsync().awaitTerminated(); } @Test @@ -217,11 +217,11 @@ public void testFencing() try { Injector injector = createInjector(0); ZKClientService zkClient = injector.getInstance(ZKClientService.class); - zkClient.startAndWait(); + zkClient.startAsync().awaitRunning(); final MessagingService messagingService = injector.getInstance(MessagingService.class); if (messagingService instanceof Service) { - ((Service) messagingService).startAndWait(); + ((Service) messagingService).startAsync().awaitRunning(); } // Shouldn't be serving request yet. @@ -246,9 +246,9 @@ public TopicId call() throws Exception { }, 10L, TimeUnit.SECONDS, 200, TimeUnit.MILLISECONDS); if (messagingService instanceof Service) { - ((Service) messagingService).stopAndWait(); + ((Service) messagingService).stopAsync().awaitTerminated(); } - zkClient.stopAndWait(); + zkClient.stopAsync().awaitTerminated(); } finally { cConf.setLong(Constants.MessagingSystem.HA_FENCING_DELAY_SECONDS, oldFencingDelay); diff --git a/cdap-tms/src/test/java/io/cdap/cdap/messaging/server/MessagingHttpServiceTest.java b/cdap-tms/src/test/java/io/cdap/cdap/messaging/server/MessagingHttpServiceTest.java index 43ff77d868b8..11a5d7286d85 100644 --- a/cdap-tms/src/test/java/io/cdap/cdap/messaging/server/MessagingHttpServiceTest.java +++ b/cdap-tms/src/test/java/io/cdap/cdap/messaging/server/MessagingHttpServiceTest.java @@ -130,13 +130,13 @@ protected void configure() { ); httpService = injector.getInstance(MessagingHttpService.class); - httpService.startAndWait(); + httpService.startAsync().awaitRunning(); client = new DefaultClientMessagingService(injector.getInstance(RemoteClientFactory.class), compressPayload); } @After public void afterTest() { - httpService.stopAndWait(); + httpService.stopAsync().awaitTerminated(); } @Test diff --git a/cdap-tms/src/test/java/io/cdap/cdap/messaging/service/ConcurrentMessageWriterTest.java b/cdap-tms/src/test/java/io/cdap/cdap/messaging/service/ConcurrentMessageWriterTest.java index 2e0184b493e3..fe86046062fc 100644 --- a/cdap-tms/src/test/java/io/cdap/cdap/messaging/service/ConcurrentMessageWriterTest.java +++ b/cdap-tms/src/test/java/io/cdap/cdap/messaging/service/ConcurrentMessageWriterTest.java @@ -232,14 +232,14 @@ public void testConcurrentWrites() throws InterruptedException, BrokenBarrierExc executor.submit(new Runnable() { @Override public void run() { - Stopwatch stopwatch = new Stopwatch(); + Stopwatch stopwatch = Stopwatch.createUnstarted(); try { barrier.await(); stopwatch.start(); for (int i = 0; i < requestPerThread; i++) { writer.persist(new TestStoreRequest(topicId, payload), metadata); } - LOG.info("Complete time for thread {} is {} ms", threadId, stopwatch.elapsedMillis()); + LOG.info("Complete time for thread {} is {} ms", threadId, stopwatch.elapsed(TimeUnit.MILLISECONDS)); } catch (Exception e) { LOG.error("Exception raised when persisting.", e); } @@ -247,13 +247,13 @@ public void run() { }); } - Stopwatch stopwatch = new Stopwatch(); + Stopwatch stopwatch = Stopwatch.createUnstarted(); barrier.await(); stopwatch.start(); executor.shutdown(); Assert.assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES)); - LOG.info("Total time passed: {} ms", stopwatch.elapsedMillis()); + LOG.info("Total time passed: {} ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); // Validate that the total number of messages written is correct List messages = testWriter.getMessages().get(topicId); diff --git a/cdap-tms/src/test/java/io/cdap/cdap/messaging/subscriber/AbstractMessagingSubscriberServiceTest.java b/cdap-tms/src/test/java/io/cdap/cdap/messaging/subscriber/AbstractMessagingSubscriberServiceTest.java index 98068afe8adf..ff2ea8661d9c 100644 --- a/cdap-tms/src/test/java/io/cdap/cdap/messaging/subscriber/AbstractMessagingSubscriberServiceTest.java +++ b/cdap-tms/src/test/java/io/cdap/cdap/messaging/subscriber/AbstractMessagingSubscriberServiceTest.java @@ -120,14 +120,14 @@ public void close() { TestMessagingSubscriberService service = new TestMessagingSubscriberService( NamespaceId.DEFAULT.topic("test"), 100, 100, 1, RetryStrategies.noRetry(), metricsContext, 100); - service.startAndWait(); + service.startAsync().awaitRunning(); //First one Assert.assertEquals(Arrays.asList(ImmutablePair.of(null, 0), ImmutablePair.of(null, 1)), processedMessages.poll(10, TimeUnit.SECONDS)); //Retry Assert.assertEquals(Arrays.asList(ImmutablePair.of(null, 0), ImmutablePair.of(null, 1)), processedMessages.poll(10, TimeUnit.SECONDS)); - service.stopAndWait(); + service.stopAsync().awaitTerminated(); } @Test @@ -158,7 +158,7 @@ public void close() { TestMessagingSubscriberService service = new TestMessagingSubscriberService( NamespaceId.DEFAULT.topic("test"), 100, 100, 1, RetryStrategies.noRetry(), metricsContext, 3); - service.startAndWait(); + service.startAsync().awaitRunning(); Assert.assertEquals(Arrays.asList(ImmutablePair.of(null, 0), ImmutablePair.of(null, 1), ImmutablePair.of(null, 2)), processedMessages.poll(10, TimeUnit.SECONDS)); Assert.assertEquals(Arrays.asList(ImmutablePair.of(null, 3)), @@ -167,7 +167,7 @@ public void close() { processedMessages.poll(10, TimeUnit.SECONDS)); Assert.assertEquals(Arrays.asList(ImmutablePair.of(null, 5), ImmutablePair.of(null, 6)), processedMessages.poll(10, TimeUnit.SECONDS)); - service.stopAndWait(); + service.stopAsync().awaitTerminated(); } /** @@ -202,10 +202,10 @@ public void close() { TestMessagingSubscriberService service = new TestMessagingSubscriberService( NamespaceId.DEFAULT.topic("test"), 100, 2, 1, RetryStrategies.noRetry(), metricsContext, 2); - service.startAndWait(); + service.startAsync().awaitRunning(); Assert.assertEquals(Arrays.asList(ImmutablePair.of(null, 0), ImmutablePair.of(null, 1)), processedMessages.poll(10, TimeUnit.SECONDS)); - service.stopAndWait(); + service.stopAsync().awaitTerminated(); } class TestMessagingSubscriberService extends AbstractMessagingSubscriberService { diff --git a/cdap-ui b/cdap-ui index 46e0c36dac1c..805e6372c1ae 160000 --- a/cdap-ui +++ b/cdap-ui @@ -1 +1 @@ -Subproject commit 46e0c36dac1c70ed45d134d4e9780dfb39a2722e +Subproject commit 805e6372c1ae9c2bd342e1957bfa7d309ce79851 diff --git a/cdap-unit-test-spark3_2.12/pom.xml b/cdap-unit-test-spark3_2.12/pom.xml index 669edb214718..3a73b4cd92e5 100644 --- a/cdap-unit-test-spark3_2.12/pom.xml +++ b/cdap-unit-test-spark3_2.12/pom.xml @@ -85,6 +85,15 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + net.alchim31.maven scala-maven-plugin diff --git a/cdap-unit-test/src/main/java/io/cdap/cdap/test/TestBase.java b/cdap-unit-test/src/main/java/io/cdap/cdap/test/TestBase.java index 914d8bd18230..733f706087e6 100644 --- a/cdap-unit-test/src/main/java/io/cdap/cdap/test/TestBase.java +++ b/cdap-unit-test/src/main/java/io/cdap/cdap/test/TestBase.java @@ -348,32 +348,42 @@ protected void configure() { ); messagingService = injector.getInstance(MessagingService.class); - if (messagingService instanceof Service) { - ((Service) messagingService).startAndWait(); + if (messagingService instanceof Service && ((Service) messagingService).state() == Service.State.NEW) { + ((Service) messagingService).startAsync().awaitRunning(); } txService = injector.getInstance(TransactionManager.class); - txService.startAndWait(); + if (txService.state() == Service.State.NEW) { + txService.startAsync().awaitRunning(); + } metadataSubscriberService = injector.getInstance(MetadataSubscriberService.class); metadataStorage = injector.getInstance(MetadataStorage.class); metadataAdmin = injector.getInstance(MetadataAdmin.class); metadataStorage.createIndex(); metadataService = injector.getInstance(MetadataService.class); - metadataService.startAndWait(); + if (metadataService.state() == Service.State.NEW) { + metadataService.startAsync().awaitRunning(); + } // Define all StructuredTable before starting any services that need StructuredTable StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class)); dsOpService = injector.getInstance(DatasetOpExecutorService.class); - dsOpService.startAndWait(); + if (dsOpService.state() == Service.State.NEW) { + dsOpService.startAsync().awaitRunning(); + } datasetService = injector.getInstance(DatasetService.class); - datasetService.startAndWait(); + if (datasetService.state() == Service.State.NEW) { + datasetService.startAsync().awaitRunning(); + } metricsCollectionService = injector.getInstance(MetricsCollectionService.class); - metricsCollectionService.startAndWait(); + if (metricsCollectionService.state() == Service.State.NEW) { + metricsCollectionService.startAsync().awaitRunning(); + } programScheduler = injector.getInstance(Scheduler.class); - if (programScheduler instanceof Service) { - ((Service) programScheduler).startAndWait(); + if (programScheduler instanceof Service && ((Service) programScheduler).state() == Service.State.NEW) { + ((Service) programScheduler).startAsync().awaitRunning(); } testManager = injector.getInstance(UnitTestManager.class); @@ -411,12 +421,18 @@ protected void configure() { messagingContext = new MultiThreadMessagingContext(messagingService); firstInit = false; previewHttpServer = injector.getInstance(PreviewHttpServer.class); - previewHttpServer.startAndWait(); + if (previewHttpServer.state() == Service.State.NEW) { + previewHttpServer.startAsync().awaitRunning(); + } fieldLineageAdmin = injector.getInstance(FieldLineageAdmin.class); lineageAdmin = injector.getInstance(LineageAdmin.class); - metadataSubscriberService.startAndWait(); + if (metadataSubscriberService.state() == Service.State.NEW) { + metadataSubscriberService.startAsync().awaitRunning(); + } artifactLocalizerService = injector.getInstance(ArtifactLocalizerService.class); - artifactLocalizerService.startAndWait(); + if (artifactLocalizerService.state() == Service.State.NEW) { + artifactLocalizerService.startAsync().awaitRunning(); + } // NOTE: As the artifact localizer client does not use service discovery for port discovery, // We need to set the port after starting the localizer service. cConf.setInt(Constants.ArtifactLocalizer.PORT, artifactLocalizerService.getPort()); @@ -426,16 +442,22 @@ protected void configure() { .setInt(Constants.ArtifactLocalizer.PORT, artifactLocalizerService.getPort()); previewRunnerManager = injector.getInstance(PreviewRunnerManager.class); - previewRunnerManager.startAndWait(); + if (previewRunnerManager.state() == Service.State.NEW) { + previewRunnerManager.startAsync().awaitRunning(); + } appFabricServer = injector.getInstance(AppFabricServer.class); - appFabricServer.startAndWait(); + if (appFabricServer.state() == Service.State.NEW) { + appFabricServer.startAsync().awaitRunning(); + } appFabricProcessorService = injector.getInstance(AppFabricProcessorService.class); - appFabricProcessorService.startAndWait(); + if (appFabricProcessorService.state() == Service.State.NEW) { + appFabricProcessorService.startAsync().awaitRunning(); + } preferencesService = injector.getInstance(PreferencesService.class); scheduler = injector.getInstance(Scheduler.class); - if (scheduler instanceof Service) { - ((Service) scheduler).startAndWait(); + if (scheduler instanceof Service && ((Service) scheduler).state() == Service.State.NEW) { + ((Service) scheduler).startAsync().awaitRunning(); } if (scheduler instanceof CoreSchedulerService) { ((CoreSchedulerService) scheduler).waitUntilFunctional(10, TimeUnit.SECONDS); @@ -604,7 +626,10 @@ private static void copyTempFile(String infileName, File outDir) throws IOExcept throw new IOException("Failed to get resource for " + infileName); } File outFile = new File(outDir, infileName); - ByteStreams.copy(Resources.newInputStreamSupplier(url), Files.newOutputStreamSupplier(outFile)); + try (java.io.InputStream in = url.openStream(); + java.io.OutputStream out = new java.io.FileOutputStream(outFile)) { + ByteStreams.copy(in, out); + } } /** @@ -617,11 +642,11 @@ public static void finish() throws Exception { } if (previewRunnerManager instanceof Service) { - previewRunnerManager.stopAndWait(); + previewRunnerManager.stopAsync().awaitTerminated(); } - artifactLocalizerService.stopAndWait(); - previewHttpServer.stopAndWait(); + artifactLocalizerService.stopAsync().awaitTerminated(); + previewHttpServer.stopAsync().awaitTerminated(); if (cConf.getBoolean(Constants.Security.Authorization.ENABLED)) { InstanceId instance = new InstanceId(cConf.get(Constants.INSTANCE_NAME)); @@ -636,23 +661,23 @@ public static void finish() throws Exception { namespaceAdmin.delete(NamespaceId.DEFAULT); if (programScheduler instanceof Service) { - ((Service) programScheduler).stopAndWait(); + ((Service) programScheduler).stopAsync().awaitTerminated(); } - metricsCollectionService.stopAndWait(); + metricsCollectionService.stopAsync().awaitTerminated(); if (scheduler instanceof Service) { - ((Service) scheduler).stopAndWait(); + ((Service) scheduler).stopAsync().awaitTerminated(); } - datasetService.stopAndWait(); - dsOpService.stopAndWait(); - metadataService.stopAndWait(); - metadataSubscriberService.stopAndWait(); - Closeables.closeQuietly(metadataStorage); - txService.stopAndWait(); + datasetService.stopAsync().awaitTerminated(); + dsOpService.stopAsync().awaitTerminated(); + metadataService.stopAsync().awaitTerminated(); + metadataSubscriberService.stopAsync().awaitTerminated(); + metadataStorage.close(); + txService.stopAsync().awaitTerminated(); if (messagingService instanceof Service) { - ((Service) messagingService).stopAndWait(); + ((Service) messagingService).stopAsync().awaitTerminated(); } - appFabricServer.stopAndWait(); - appFabricProcessorService.stopAndWait(); + appFabricServer.stopAsync().awaitTerminated(); + appFabricProcessorService.stopAsync().awaitTerminated(); } protected MetricsManager getMetricsManager() { diff --git a/cdap-unit-test/src/test/java/io/cdap/cdap/service/FileUploadApp.java b/cdap-unit-test/src/test/java/io/cdap/cdap/service/FileUploadApp.java index e53c6663a27a..de4219fe356b 100644 --- a/cdap-unit-test/src/test/java/io/cdap/cdap/service/FileUploadApp.java +++ b/cdap-unit-test/src/test/java/io/cdap/cdap/service/FileUploadApp.java @@ -17,7 +17,7 @@ package io.cdap.cdap.service; import com.google.common.base.Throwables; -import com.google.common.io.Closeables; + import io.cdap.cdap.api.Transactional; import io.cdap.cdap.api.TxRunnable; import io.cdap.cdap.api.app.AbstractApplication; @@ -134,7 +134,11 @@ public void onFinish(HttpServiceResponder responder) throws Exception { @Override public void onError(HttpServiceResponder responder, Throwable failureCause) { try { - Closeables.closeQuietly(channel); + try { + channel.close(); + } catch (IOException e) { + // ignore + } partitionDir.delete(true); } catch (IOException e) { // Nothing much can be done. diff --git a/cdap-unit-test/src/test/java/io/cdap/cdap/service/FileUploadServiceTestRun.java b/cdap-unit-test/src/test/java/io/cdap/cdap/service/FileUploadServiceTestRun.java index b90b6b995e77..7ee29d580755 100644 --- a/cdap-unit-test/src/test/java/io/cdap/cdap/service/FileUploadServiceTestRun.java +++ b/cdap-unit-test/src/test/java/io/cdap/cdap/service/FileUploadServiceTestRun.java @@ -94,7 +94,9 @@ public void testFileUploadService() throws Exception { // There should be one file under the partition directory List locations = partition.getLocation().list(); Assert.assertEquals(1, locations.size()); - Assert.assertArrayEquals(content, ByteStreams.toByteArray(Locations.newInputSupplier(locations.get(0)))); + try (java.io.InputStream is = locations.get(0).getInputStream()) { + Assert.assertArrayEquals(content, ByteStreams.toByteArray(is)); + } // Verify the tracking table of chunks sizes KeyValueTable trackingTable = (KeyValueTable) getDataset(FileUploadApp.KV_TABLE_NAME).get(); diff --git a/cdap-unit-test/src/test/java/io/cdap/cdap/test/messaging/MessagingApp.java b/cdap-unit-test/src/test/java/io/cdap/cdap/test/messaging/MessagingApp.java index 36d3922144bd..a935e17e677e 100644 --- a/cdap-unit-test/src/test/java/io/cdap/cdap/test/messaging/MessagingApp.java +++ b/cdap-unit-test/src/test/java/io/cdap/cdap/test/messaging/MessagingApp.java @@ -60,9 +60,9 @@ public void configure() { private static Message fetchMessage(MessageFetcher fetcher, String namespace, String topic, @Nullable String afterMessageId, long timeout, TimeUnit unit) throws Exception { CloseableIterator iterator = fetcher.fetch(namespace, topic, 1, afterMessageId); - Stopwatch stopwatch = new Stopwatch().start(); + Stopwatch stopwatch = Stopwatch.createStarted(); try { - while (!iterator.hasNext() && stopwatch.elapsedTime(unit) < timeout) { + while (!iterator.hasNext() && stopwatch.elapsed(unit) < timeout) { TimeUnit.MILLISECONDS.sleep(100); iterator = fetcher.fetch(namespace, topic, 1, afterMessageId); } diff --git a/cdap-unit-test/src/test/scala/io/cdap/cdap/spark/app/ForkSpark.scala b/cdap-unit-test/src/test/scala/io/cdap/cdap/spark/app/ForkSpark.scala index 2b29fc135207..0820d88ef316 100644 --- a/cdap-unit-test/src/test/scala/io/cdap/cdap/spark/app/ForkSpark.scala +++ b/cdap-unit-test/src/test/scala/io/cdap/cdap/spark/app/ForkSpark.scala @@ -47,8 +47,8 @@ class ForkSpark(name: String) extends AbstractSpark with SparkMain { val file = new File(barrierDir, sec.getRunId.getId) require(file.createNewFile()) val branchSize = getBranchSize(sec) - val stopWatch = new Stopwatch().start() - while (barrierDir.list().length < branchSize && stopWatch.elapsedTime(TimeUnit.SECONDS) < 10) { + val stopWatch = Stopwatch.createStarted() + while (barrierDir.list().length < branchSize && stopWatch.elapsed(TimeUnit.SECONDS) < 10) { TimeUnit.MILLISECONDS.sleep(100) } diff --git a/cdap-unit-test/src/test/scala/io/cdap/cdap/test/messaging/MessagingSpark.scala b/cdap-unit-test/src/test/scala/io/cdap/cdap/test/messaging/MessagingSpark.scala index 3822efa84bb2..3778c6a6cdf8 100644 --- a/cdap-unit-test/src/test/scala/io/cdap/cdap/test/messaging/MessagingSpark.scala +++ b/cdap-unit-test/src/test/scala/io/cdap/cdap/test/messaging/MessagingSpark.scala @@ -82,9 +82,9 @@ class MessagingSpark extends AbstractSpark with SparkMain { private def fetchMessage(fetcher: MessageFetcher, namespace: String, topic: String, afterMessageId: Option[String], timeout: Long, unit: TimeUnit): Message = { var iterator = fetcher.fetch(namespace, topic, 1, afterMessageId.orNull) - val stopwatch = new Stopwatch().start + val stopwatch = Stopwatch.createStarted() try { - while (!iterator.hasNext && stopwatch.elapsedTime(unit) < timeout) { + while (!iterator.hasNext && stopwatch.elapsed(unit) < timeout) { TimeUnit.MILLISECONDS.sleep(100) iterator = fetcher.fetch(namespace, topic, 1, afterMessageId.orNull) } diff --git a/cdap-watchdog-api/pom.xml b/cdap-watchdog-api/pom.xml index 88fd6d905434..dbb581f97b64 100644 --- a/cdap-watchdog-api/pom.xml +++ b/cdap-watchdog-api/pom.xml @@ -60,4 +60,18 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 17 + 17 + + + + + diff --git a/cdap-watchdog-api/src/main/java/io/cdap/cdap/api/metrics/MetricDataQuery.java b/cdap-watchdog-api/src/main/java/io/cdap/cdap/api/metrics/MetricDataQuery.java index bae379b8e68f..261b55c76739 100644 --- a/cdap-watchdog-api/src/main/java/io/cdap/cdap/api/metrics/MetricDataQuery.java +++ b/cdap-watchdog-api/src/main/java/io/cdap/cdap/api/metrics/MetricDataQuery.java @@ -26,15 +26,13 @@ /** * Defines a query to perform on {@link MetricStore} data. - *

* Though limited currently in functionality, you can map {@link MetricDataQuery} to the following * statement: * 
- * SELECT count('read.ops')                                     << metric name and aggregation function
+ * SELECT count('read.ops')                                     << metric name and aggregation function
  * FROM Cube
- * GROUP BY dataset,                                            << groupByTags
- * WHERE namespace='ns1' AND app='myApp' AND program='myFlow'   << sliceByTags
- *
+ * GROUP BY dataset,                                            << groupByTags
+ * WHERE namespace='ns1' AND app='myApp' AND program='myFlow'   << sliceByTags
  *
*/ public final class MetricDataQuery { diff --git a/cdap-watchdog/pom.xml b/cdap-watchdog/pom.xml index c1aec78b34ff..1899f3ed7cf2 100644 --- a/cdap-watchdog/pom.xml +++ b/cdap-watchdog/pom.xml @@ -153,4 +153,18 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 17 + 17 + + + + + diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/LogMessage.java b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/LogMessage.java index b2f6d5a1918d..42552034c0c3 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/LogMessage.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/LogMessage.java @@ -20,7 +20,7 @@ import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.classic.spi.IThrowableProxy; import ch.qos.logback.classic.spi.LoggerContextVO; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import io.cdap.cdap.common.logging.LoggingContext; import java.util.Map; import org.slf4j.Marker; @@ -130,7 +130,7 @@ public void prepareForDeferredProcessing() { @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("loggingEvent", loggingEvent) .add("loggingContext", loggingContext) .toString(); diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/kafka/KafkaLogAppender.java b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/kafka/KafkaLogAppender.java index 0cc13bbafc93..6075752a97d5 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/kafka/KafkaLogAppender.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/kafka/KafkaLogAppender.java @@ -50,8 +50,8 @@ public final class KafkaLogAppender extends LogAppender { public void start() { KafkaLogPublisher publisher = new KafkaLogPublisher(cConf); Optional.ofNullable(kafkaLogPublisher.getAndSet(publisher)) - .ifPresent(KafkaLogPublisher::stopAndWait); - publisher.startAndWait(); + .ifPresent(p -> p.stopAsync().awaitTerminated()); + publisher.startAsync().awaitRunning(); addInfo("Successfully started KafkaLogAppender."); super.start(); } @@ -60,7 +60,7 @@ public void start() { public void stop() { super.stop(); Optional.ofNullable(kafkaLogPublisher.getAndSet(null)) - .ifPresent(KafkaLogPublisher::stopAndWait); + .ifPresent(p -> p.stopAsync().awaitTerminated()); } @Override diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/kafka/StringPartitioner.java b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/kafka/StringPartitioner.java index 838bdadcf2e4..64a97ab7c760 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/kafka/StringPartitioner.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/kafka/StringPartitioner.java @@ -47,6 +47,7 @@ public StringPartitioner(CConfiguration cConf) { @Override public int partition(Object key, int numPartitions) { - return Math.abs(Hashing.md5().hashString(key.toString()).asInt()) % this.numPartitions; + return Math.abs(Hashing.md5().hashString(key.toString(), + java.nio.charset.StandardCharsets.UTF_8).asInt()) % this.numPartitions; } } diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/remote/RemoteLogAppender.java b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/remote/RemoteLogAppender.java index e1a79e182df8..34749d2073ff 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/remote/RemoteLogAppender.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/remote/RemoteLogAppender.java @@ -75,8 +75,8 @@ public RemoteLogAppender(CConfiguration cConf, RemoteClientFactory remoteClientF public void start() { RemoteLogPublisher publisher = new RemoteLogPublisher(cConf, remoteClientFactory); Optional.ofNullable(this.publisher.getAndSet(publisher)) - .ifPresent(RemoteLogPublisher::stopAndWait); - publisher.startAndWait(); + .ifPresent(p -> p.stopAsync().awaitTerminated()); + publisher.startAsync().awaitRunning(); addInfo("Successfully started " + APPENDER_NAME); super.start(); } @@ -84,7 +84,7 @@ public void start() { @Override public void stop() { super.stop(); - Optional.ofNullable(this.publisher.getAndSet(null)).ifPresent(RemoteLogPublisher::stopAndWait); + Optional.ofNullable(this.publisher.getAndSet(null)).ifPresent(p -> p.stopAsync().awaitTerminated()); addInfo("Successfully stopped " + APPENDER_NAME); } @@ -178,7 +178,7 @@ protected void logError(String errorMessage, Exception exception) { * do not want kafka dependencies in this class, this method is added here. */ private static int partition(String key, int numPartitions) { - return Math.abs(Hashing.md5().hashString(key).asInt()) % numPartitions; + return Math.abs(Hashing.md5().hashString(key, java.nio.charset.StandardCharsets.UTF_8).asInt()) % numPartitions; } private void encodeEvents(OutputStream os, DatumWriter> datumWriter, diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/system/LogFileManager.java b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/system/LogFileManager.java index 29cf129f60eb..5414c0142345 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/system/LogFileManager.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/system/LogFileManager.java @@ -119,7 +119,13 @@ public void close() throws IOException { location.getLocation()); } catch (Throwable e) { // delete created file as there was exception while writing meta data - Closeables.closeQuietly(logFileOutputStream); + if (logFileOutputStream instanceof AutoCloseable) { + try { + ((AutoCloseable) logFileOutputStream).close(); + } catch (Exception ex) { + // Ignore + } + } Locations.deleteQuietly(location.getLocation()); throw new IOException(e); } @@ -173,7 +179,13 @@ public void close() { outputStreamMap.clear(); for (LogFileOutputStream stream : streams) { - Closeables.closeQuietly(stream); + if (stream instanceof AutoCloseable) { + try { + ((AutoCloseable) stream).close(); + } catch (Exception e) { + // Ignore + } + } } } diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/system/LogFileOutputStream.java b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/system/LogFileOutputStream.java index d505fe7e2cf8..d64655e34ce0 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/system/LogFileOutputStream.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/system/LogFileOutputStream.java @@ -72,8 +72,20 @@ class LogFileOutputStream implements Closeable, Flushable, Syncable { this.createTime = createTime; this.fileSize = 0; } catch (IOException e) { - Closeables.closeQuietly(outputStream); - Closeables.closeQuietly(dataFileWriter); + if (outputStream != null) { + try { + outputStream.close(); + } catch (IOException ioe) { + // Ignore + } + } + if (dataFileWriter != null) { + try { + dataFileWriter.close(); + } catch (IOException ioe) { + // Ignore + } + } throw e; } } diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/tms/TMSLogAppender.java b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/tms/TMSLogAppender.java index be8f229a4195..87cceb006a6b 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/tms/TMSLogAppender.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/appender/tms/TMSLogAppender.java @@ -66,8 +66,8 @@ public class TMSLogAppender extends LogAppender { public void start() { TMSLogPublisher publisher = new TMSLogPublisher(cConf, messagingService); Optional.ofNullable(tmsLogPublisher.getAndSet(publisher)) - .ifPresent(TMSLogPublisher::stopAndWait); - publisher.startAndWait(); + .ifPresent(p -> p.stopAsync().awaitTerminated()); + publisher.startAsync().awaitRunning(); addInfo("Successfully started " + APPENDER_NAME); super.start(); } @@ -75,7 +75,7 @@ public void start() { @Override public void stop() { super.stop(); - Optional.ofNullable(tmsLogPublisher.getAndSet(null)).ifPresent(TMSLogPublisher::stopAndWait); + Optional.ofNullable(tmsLogPublisher.getAndSet(null)).ifPresent(p -> p.stopAsync().awaitTerminated()); addInfo("Successfully stopped " + APPENDER_NAME); } @@ -95,7 +95,8 @@ protected void appendEvent(LogMessage logMessage) { // in Standalone @VisibleForTesting static int partition(Object key, int numPartitions) { - return Math.abs(Hashing.md5().hashString(key.toString()).asInt()) % numPartitions; + return Math.abs(Hashing.md5().hashString(key.toString(), + java.nio.charset.StandardCharsets.UTF_8).asInt()) % numPartitions; } /** diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/filter/AndFilter.java b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/filter/AndFilter.java index 777a556685cb..281efca9a5a9 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/filter/AndFilter.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/filter/AndFilter.java @@ -17,7 +17,7 @@ package io.cdap.cdap.logging.filter; import ch.qos.logback.classic.spi.ILoggingEvent; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; import java.util.List; @@ -44,7 +44,7 @@ public boolean match(ILoggingEvent event) { @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("expressions", expressions) .toString(); } diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/filter/MdcExpression.java b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/filter/MdcExpression.java index 9dfcf8b1620f..71c633752a32 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/filter/MdcExpression.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/filter/MdcExpression.java @@ -17,7 +17,7 @@ package io.cdap.cdap.logging.filter; import ch.qos.logback.classic.spi.ILoggingEvent; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; /** * Represents an expression that can match a key,value in MDC. @@ -48,7 +48,7 @@ public String getValue() { @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("key", key) .add("value", value) .toString(); diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/filter/OrFilter.java b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/filter/OrFilter.java index 9a993b36a3f7..29952c8f538f 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/filter/OrFilter.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/filter/OrFilter.java @@ -17,7 +17,7 @@ package io.cdap.cdap.logging.filter; import ch.qos.logback.classic.spi.ILoggingEvent; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; import java.util.List; @@ -44,7 +44,7 @@ public boolean match(ILoggingEvent event) { @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("expressions", expressions) .toString(); } diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/framework/distributed/DistributedLogFramework.java b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/framework/distributed/DistributedLogFramework.java index 871e0d2cd187..7b0f1f4e443a 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/framework/distributed/DistributedLogFramework.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/framework/distributed/DistributedLogFramework.java @@ -112,13 +112,47 @@ protected Service createService(Set partitions) { @Override protected void startUp() throws Exception { // Starts all pipeline - validateAllFutures(Iterables.transform(pipelines, Service::start)); + List> startFutures = new java.util.ArrayList<>(); + for (Service pipeline : pipelines) { + final com.google.common.util.concurrent.SettableFuture future = + com.google.common.util.concurrent.SettableFuture.create(); + pipeline.addListener(new Service.Listener() { + @Override + public void running() { + future.set(null); + } + @Override + public void failed(Service.State from, Throwable failure) { + future.setException(failure); + } + }, com.google.common.util.concurrent.MoreExecutors.directExecutor()); + pipeline.startAsync(); + startFutures.add(future); + } + validateAllFutures(startFutures); } @Override protected void shutDown() throws Exception { // Stops all pipeline - validateAllFutures(Iterables.transform(pipelines, Service::stop)); + List> stopFutures = new java.util.ArrayList<>(); + for (Service pipeline : pipelines) { + final com.google.common.util.concurrent.SettableFuture future = + com.google.common.util.concurrent.SettableFuture.create(); + pipeline.addListener(new Service.Listener() { + @Override + public void terminated(Service.State from) { + future.set(null); + } + @Override + public void failed(Service.State from, Throwable failure) { + future.setException(failure); + } + }, com.google.common.util.concurrent.MoreExecutors.directExecutor()); + pipeline.stopAsync(); + stopFutures.add(future); + } + validateAllFutures(stopFutures); } }; } diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/framework/local/LocalLogAppender.java b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/framework/local/LocalLogAppender.java index 1e4b0b91b20f..5c1a2c300be3 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/framework/local/LocalLogAppender.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/framework/local/LocalLogAppender.java @@ -96,12 +96,12 @@ public void start() { spec.getContext().getMetricsContext(), spec.getContext().getInstanceId()); LocalLogProcessorPipeline pipeline = new LocalLogProcessorPipeline(context, syncIntervalMillis); - pipeline.startAndWait(); + pipeline.startAsync().awaitRunning(); pipelineThreads.add(pipeline.getAppenderThread()); pipelines.add(pipeline); } - this.pipelines.getAndSet(pipelines).forEach(LocalLogProcessorPipeline::stopAndWait); + this.pipelines.getAndSet(pipelines).forEach(p -> p.stopAsync().awaitTerminated()); this.pipelineThreads.set(pipelineThreads); super.start(); @@ -113,7 +113,7 @@ public void stop() { super.stop(); for (LocalLogProcessorPipeline pipeline : pipelines.getAndSet(Collections.emptyList())) { try { - pipeline.stopAndWait(); + pipeline.stopAsync().awaitTerminated(); } catch (Throwable t) { addError("Exception raised when stopping log processing pipeline " + pipeline.getName(), t); } diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/gateway/handlers/AbstractChunkedCallback.java b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/gateway/handlers/AbstractChunkedCallback.java index 7638ea8c7c4b..cabd6aff1d1e 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/gateway/handlers/AbstractChunkedCallback.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/gateway/handlers/AbstractChunkedCallback.java @@ -112,7 +112,13 @@ public void close() { // Just log the error as debug. LOG.debug("Failed to send chunk", e); } finally { - Closeables.closeQuietly(chunkResponder); + if (chunkResponder instanceof AutoCloseable) { + try { + ((AutoCloseable) chunkResponder).close(); + } catch (Exception e) { + // Ignore + } + } } } diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/gateway/handlers/RemoteLogsFetcher.java b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/gateway/handlers/RemoteLogsFetcher.java index 303a1dccf150..d98b80259ac8 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/gateway/handlers/RemoteLogsFetcher.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/gateway/handlers/RemoteLogsFetcher.java @@ -114,7 +114,7 @@ public boolean onReceived(ByteBuffer buffer) { @Override public void onFinished() { - Closeables.closeQuietly(channel); + // Channel will be closed by try-with-resources block in execute method } }).build(); remoteClient.executeStreamingRequest(request); diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/logbuffer/LogBufferService.java b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/logbuffer/LogBufferService.java index 99e475757a18..f48f5ddcd663 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/logbuffer/LogBufferService.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/logbuffer/LogBufferService.java @@ -102,7 +102,24 @@ protected void startUp() throws Exception { // load log pipelines List bufferPipelines = loadLogPipelines(); // start all the log pipelines - validateAllFutures(Iterables.transform(pipelines, Service::start)); + List> startFutures = new java.util.ArrayList<>(); + for (Service pipeline : pipelines) { + final com.google.common.util.concurrent.SettableFuture future = + com.google.common.util.concurrent.SettableFuture.create(); + pipeline.addListener(new Service.Listener() { + @Override + public void running() { + future.set(null); + } + @Override + public void failed(Service.State from, Throwable failure) { + future.setException(failure); + } + }, com.google.common.util.concurrent.MoreExecutors.directExecutor()); + pipeline.startAsync(); + startFutures.add(future); + } + validateAllFutures(startFutures); // recovery service and http handler will send log events to log pipelines. In order to avoid deleting file while // reading them in recovery service, we will pass in an atomic boolean will be set to true by recovery service @@ -111,7 +128,7 @@ protected void startUp() throws Exception { // start log recovery service to recover all the pending logs. recoveryService = new LogBufferRecoveryService(cConf, bufferPipelines, checkpointManagers, startCleanup); - recoveryService.startAndWait(); + recoveryService.startAsync().awaitRunning(); // create concurrent writer ConcurrentLogBufferWriter concurrentWriter = new ConcurrentLogBufferWriter(cConf, @@ -231,9 +248,26 @@ private void stopAllServices() throws Exception { httpService.stop(); } if (recoveryService != null) { - recoveryService.stopAndWait(); + recoveryService.stopAsync().awaitTerminated(); } // Stops all pipeline - validateAllFutures(Iterables.transform(pipelines, Service::stop)); + List> stopFutures = new java.util.ArrayList<>(); + for (Service pipeline : pipelines) { + final com.google.common.util.concurrent.SettableFuture future = + com.google.common.util.concurrent.SettableFuture.create(); + pipeline.addListener(new Service.Listener() { + @Override + public void terminated(Service.State from) { + future.set(null); + } + @Override + public void failed(Service.State from, Throwable failure) { + future.setException(failure); + } + }, com.google.common.util.concurrent.MoreExecutors.directExecutor()); + pipeline.stopAsync(); + stopFutures.add(future); + } + validateAllFutures(stopFutures); } } diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/logbuffer/LogBufferWriter.java b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/logbuffer/LogBufferWriter.java index b21cc2994013..330774c5b467 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/logbuffer/LogBufferWriter.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/logbuffer/LogBufferWriter.java @@ -154,7 +154,13 @@ public void close() throws IOException { LOG.warn("Error while flushing log buffer output stream.", e); } - Closeables.closeQuietly(currOutputStream); + if (currOutputStream != null) { + try { + currOutputStream.close(); + } catch (IOException e) { + // Ignore + } + } executorService.shutdown(); } @@ -184,7 +190,13 @@ private long getNextFileId(File baseDir) { private Location rotateFile(OutputStream currOutputStream) throws IOException { currOutputStream.flush(); // close current location output stream - Closeables.closeQuietly(currOutputStream); + if (currOutputStream != null) { + try { + currOutputStream.close(); + } catch (IOException e) { + // Ignore + } + } writtenBytes = 0; currOffset = 0; diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/logbuffer/recover/LogBufferRecoveryService.java b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/logbuffer/recover/LogBufferRecoveryService.java index 4888002bb813..4c47d5719cb3 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/logbuffer/recover/LogBufferRecoveryService.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/logbuffer/recover/LogBufferRecoveryService.java @@ -127,7 +127,7 @@ protected void triggerShutdown() { } @Override - protected String getServiceName() { + protected String serviceName() { return SERVICE_NAME; } diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/meta/Checkpoint.java b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/meta/Checkpoint.java index 33db81b178a7..8a8633ae5458 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/meta/Checkpoint.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/meta/Checkpoint.java @@ -16,7 +16,7 @@ package io.cdap.cdap.logging.meta; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; /** * Represents a checkpoint that can be saved when reading logs. @@ -52,7 +52,7 @@ public long getMaxEventTime() { @Override public String toString() { - return Objects.toStringHelper(this) + return com.google.common.base.MoreObjects.toStringHelper(this) .add("Offset", offset) .add("maxEventTime", maxEventTime) .toString(); diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/pipeline/kafka/KafkaLogProcessorPipeline.java b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/pipeline/kafka/KafkaLogProcessorPipeline.java index 2e4bc7b24261..65408bbb3c72 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/pipeline/kafka/KafkaLogProcessorPipeline.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/pipeline/kafka/KafkaLogProcessorPipeline.java @@ -223,7 +223,7 @@ protected void shutDown() throws Exception { } @Override - protected String getServiceName() { + protected String serviceName() { return "LogPipeline-" + name; } diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/pipeline/logbuffer/LogBufferProcessorPipeline.java b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/pipeline/logbuffer/LogBufferProcessorPipeline.java index 15332657b026..cb1088e10627 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/pipeline/logbuffer/LogBufferProcessorPipeline.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/pipeline/logbuffer/LogBufferProcessorPipeline.java @@ -139,7 +139,7 @@ protected void shutDown() throws Exception { } @Override - protected String getServiceName() { + protected String serviceName() { return "LogPipeline-" + name; } diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/plugins/LocationManager.java b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/plugins/LocationManager.java index a95186d7be9e..39f0059886e7 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/plugins/LocationManager.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/plugins/LocationManager.java @@ -175,7 +175,13 @@ public void close() { for (LocationOutputStream locationOutputStream : locations) { // we do not want to throw any exception rather close all the open output streams. so close quietly - Closeables.closeQuietly(locationOutputStream); + if (locationOutputStream instanceof AutoCloseable) { + try { + ((AutoCloseable) locationOutputStream).close(); + } catch (Exception e) { + // Ignore + } + } } activeLocations.clear(); diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/read/LogOffset.java b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/read/LogOffset.java index 90bfbeb8b087..e7de5c993665 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/read/LogOffset.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/read/LogOffset.java @@ -16,7 +16,7 @@ package io.cdap.cdap.logging.read; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; /** * Represents log offset containing Kafka offset and time of logging event. @@ -45,7 +45,7 @@ public long getTime() { @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("kafkaOffset", kafkaOffset) .add("time", time) .toString(); diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/read/ReadRange.java b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/read/ReadRange.java index 2d3958da0708..8f299588518c 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/read/ReadRange.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/read/ReadRange.java @@ -16,7 +16,7 @@ package io.cdap.cdap.logging.read; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; /** * Boundary of a log read request. @@ -64,7 +64,7 @@ public static ReadRange createToRange(LogOffset logOffset) { @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("fromMillis", fromMillis) .add("toMillis", toMillis) .add("kafkaOffset", kafkaOffset) diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/service/LogSaverStatusService.java b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/service/LogSaverStatusService.java index 4e1192df1888..813d6c4148f3 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/service/LogSaverStatusService.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/service/LogSaverStatusService.java @@ -16,7 +16,7 @@ package io.cdap.cdap.logging.service; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.util.concurrent.AbstractIdleService; import com.google.inject.Inject; import com.google.inject.name.Named; @@ -90,7 +90,7 @@ protected void shutDown() throws Exception { @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("bindAddress", httpService.getBindAddress()) .toString(); } diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/write/LogLocation.java b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/write/LogLocation.java index b67452fd1988..893bb67c8d71 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/write/LogLocation.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/write/LogLocation.java @@ -407,12 +407,12 @@ private static final class LocationSeekableInput implements SeekableInput { this.is = impersonator.doAs(namespaceId, new Callable() { @Override public SeekableInputStream call() throws Exception { - return Locations.newInputSupplier(location).getInput(); + return Locations.newInputSupplier(location).get(); } }); } else { // impersonation is not required for V1 version. - this.is = Locations.newInputSupplier(location).getInput(); + this.is = Locations.newInputSupplier(location).get(); } } catch (IOException e) { throw e; diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/metrics/collect/LocalMetricsCollectionService.java b/cdap-watchdog/src/main/java/io/cdap/cdap/metrics/collect/LocalMetricsCollectionService.java index 1470953249cf..ef0b68c0ee0f 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/metrics/collect/LocalMetricsCollectionService.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/metrics/collect/LocalMetricsCollectionService.java @@ -88,11 +88,11 @@ protected void startUp() throws Exception { IntStream.range(0, cConf.getInt(Constants.Metrics.MESSAGING_TOPIC_NUM)).boxed() .collect(Collectors.toSet()), getContext(METRICS_PROCESSOR_CONTEXT), 0); - messagingMetricsProcessor.startAndWait(); + messagingMetricsProcessor.startAsync().awaitRunning(); } // The local metrics store do not have ttl, so start the clean up service - metricsCleanUpService.startAndWait(); + metricsCleanUpService.startAsync().awaitRunning(); } @Override @@ -101,7 +101,7 @@ protected void shutDown() throws Exception { Exception failure = null; try { if (messagingMetricsProcessor != null) { - messagingMetricsProcessor.stopAndWait(); + messagingMetricsProcessor.stopAsync().awaitTerminated(); } } catch (Exception e) { failure = e; @@ -120,7 +120,7 @@ protected void shutDown() throws Exception { // Shutdown the clean up service try { - metricsCleanUpService.stopAndWait(); + metricsCleanUpService.stopAsync().awaitTerminated(); } catch (Exception e) { if (failure != null) { failure.addSuppressed(e); diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/metrics/process/MessagingMetricsProcessorManagerService.java b/cdap-watchdog/src/main/java/io/cdap/cdap/metrics/process/MessagingMetricsProcessorManagerService.java index 7df8daf9d5db..6e0c500f4fc6 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/metrics/process/MessagingMetricsProcessorManagerService.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/metrics/process/MessagingMetricsProcessorManagerService.java @@ -144,7 +144,7 @@ protected void startUp() throws Exception { } for (MessagingMetricsProcessorService processorService : metricsProcessorServices) { - processorService.startAndWait(); + processorService.startAsync().awaitRunning(); } } @@ -192,7 +192,7 @@ protected void shutDown() throws Exception { Exception exceptions = new Exception(); for (MessagingMetricsProcessorService processorService : metricsProcessorServices) { try { - processorService.stopAndWait(); + processorService.stopAsync().awaitTerminated(); for (MetricsWriter metricsWriter : metricsWriters) { metricsWriter.close(); } diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/metrics/process/MetricsProcessorStatusService.java b/cdap-watchdog/src/main/java/io/cdap/cdap/metrics/process/MetricsProcessorStatusService.java index 8c343826b2ef..09044a9c85b7 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/metrics/process/MetricsProcessorStatusService.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/metrics/process/MetricsProcessorStatusService.java @@ -16,7 +16,7 @@ package io.cdap.cdap.metrics.process; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.util.concurrent.AbstractIdleService; import com.google.inject.Inject; import com.google.inject.name.Named; @@ -99,7 +99,7 @@ protected void shutDown() throws Exception { @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("bindAddress", httpService.getBindAddress()) .toString(); } diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/metrics/query/TimeseriesId.java b/cdap-watchdog/src/main/java/io/cdap/cdap/metrics/query/TimeseriesId.java index d56ce0c080ab..066c4beb4e27 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/metrics/query/TimeseriesId.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/metrics/query/TimeseriesId.java @@ -16,7 +16,7 @@ package io.cdap.cdap.metrics.query; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; /** * class to identify a unique timeseries, which is a 4 tuple of context, metric, tag, and runid. @@ -41,20 +41,20 @@ public boolean equals(Object o) { return false; } TimeseriesId other = (TimeseriesId) o; - return Objects.equal(context, other.context) - && Objects.equal(metric, other.metric) - && Objects.equal(tag, other.tag) - && Objects.equal(runId, other.runId); + return java.util.Objects.equals(context, other.context) + && java.util.Objects.equals(metric, other.metric) + && java.util.Objects.equals(tag, other.tag) + && java.util.Objects.equals(runId, other.runId); } @Override public int hashCode() { - return Objects.hashCode(context, metric, tag, runId); + return java.util.Objects.hash(context, metric, tag, runId); } @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("context", context) .add("metric", metric) .add("tag", tag) diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/ErrorLogsClassifierTest.java b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/ErrorLogsClassifierTest.java index 6afa86b36245..2b5e535937da 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/ErrorLogsClassifierTest.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/ErrorLogsClassifierTest.java @@ -70,13 +70,13 @@ public void testClassifyLogsWithFailureDetailsProvider() { List metricValuesList = new ArrayList<>(); MetricsCollectionService mockMetricsCollectionService = getMockCollectionService(metricValuesList); - mockMetricsCollectionService.startAndWait(); + mockMetricsCollectionService.startAsync().awaitRunning(); CConfiguration cConf = Mockito.mock(CConfiguration.class); ErrorLogsClassifier classifier = new ErrorLogsClassifier(cConf, mockMetricsCollectionService); classifier.classify(closeableIterator, responder, "namespace", "program", "app", "run"); List responses = GSON.fromJson(responder.getResponseContentAsString(), LIST_TYPE); - mockMetricsCollectionService.stopAndWait(); + mockMetricsCollectionService.stopAsync().awaitTerminated(); Assert.assertEquals(1, responses.size()); Assert.assertEquals("stageName", responses.get(0).getStageName()); Assert.assertEquals("errorCategory-'stageName'", responses.get(0).getErrorCategory()); @@ -98,7 +98,7 @@ public void testClassifyLogsWithRuleBasedClassification() { List metricValuesList = new ArrayList<>(); MetricsCollectionService mockMetricsCollectionService = getMockCollectionService(metricValuesList); - mockMetricsCollectionService.startAndWait(); + mockMetricsCollectionService.startAsync().awaitRunning(); CConfiguration cConf = Mockito.mock(CConfiguration.class); ErrorLogsClassifier classifier = new ErrorLogsClassifier(cConf, mockMetricsCollectionService); LogEvent logEvent3 = new LogEvent(getEvent3(IllegalArgumentException.class.getName()), @@ -112,10 +112,10 @@ public void testClassifyLogsWithRuleBasedClassification() { Mockito.when(spy.getRuleList()).thenReturn(getRulesList()); Mockito.doCallRealMethod().when(spy).classify(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); - mockMetricsCollectionService.startAndWait(); + mockMetricsCollectionService.startAsync().awaitRunning(); CloseableIterator closeableIterator = getCloseableIterator(events.iterator()); spy.classify(closeableIterator, responder, "namespace", "program", "app", "run2"); - mockMetricsCollectionService.stopAndWait(); + mockMetricsCollectionService.stopAsync().awaitTerminated(); List responses = GSON.fromJson(responder.getResponseContentAsString(), LIST_TYPE); Assert.assertEquals(1, responses.size()); diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/ErrorClassificationLoggingTest.java b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/ErrorClassificationLoggingTest.java index 2c488481b870..18644d02d99c 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/ErrorClassificationLoggingTest.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/ErrorClassificationLoggingTest.java @@ -147,6 +147,6 @@ public void testErrorClassificationTagsArePresentWithWrappedStageException() { @AfterClass public static void cleanUp() { - txManager.stopAndWait(); + txManager.stopAsync().awaitTerminated(); } } diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/ErrorTagProviderLoggingTest.java b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/ErrorTagProviderLoggingTest.java index 0aaf03ad451c..5288ca5a2bec 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/ErrorTagProviderLoggingTest.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/ErrorTagProviderLoggingTest.java @@ -104,7 +104,7 @@ public void testErrorCodeProviderException() { @AfterClass public static void cleanUp() throws Exception { - txManager.stopAndWait(); + txManager.stopAsync().awaitTerminated(); } } diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/LocalLogAppenderResilientTest.java b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/LocalLogAppenderResilientTest.java index 0258033b4c17..324176207b97 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/LocalLogAppenderResilientTest.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/LocalLogAppenderResilientTest.java @@ -134,12 +134,12 @@ protected void configure() { }); TransactionManager txManager = injector.getInstance(TransactionManager.class); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class)); DatasetOpExecutorService opExecutorService = injector .getInstance(DatasetOpExecutorService.class); - opExecutorService.startAndWait(); + opExecutorService.startAsync().awaitRunning(); // Start the logging before starting the service. LoggingContextAccessor @@ -187,7 +187,7 @@ public void addStatusEvent(Status status) { // Start dataset service, wait for it to be discoverable DatasetService dsService = injector.getInstance(DatasetService.class); - dsService.startAndWait(); + dsService.startAsync().awaitRunning(); final CountDownLatch startLatch = new CountDownLatch(1); DiscoveryServiceClient discoveryClient = injector.getInstance(DiscoveryServiceClient.class); diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/LoggingTester.java b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/LoggingTester.java index b88a7a88506c..196ffba053cf 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/LoggingTester.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/LoggingTester.java @@ -113,7 +113,7 @@ protected void configure() { public static TransactionManager createTransactionManager(Injector injector) throws IOException { TransactionManager txManager = injector.getInstance(TransactionManager.class); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); return txManager; } diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/TestDistributedLogReader.java b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/TestDistributedLogReader.java index ce7a499e6dd6..8f5eab5e680f 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/TestDistributedLogReader.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/TestDistributedLogReader.java @@ -97,7 +97,7 @@ public static void setUpContext() throws Exception { stringPartitioner.partition(LOGGING_CONTEXT_KAFKA.getLogPartition(), -1)); txManager = injector.getInstance(TransactionManager.class); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class)); @@ -152,7 +152,7 @@ public static void setUpContext() throws Exception { @AfterClass public static void cleanUp() throws Exception { InMemoryTableService.reset(); - txManager.stopAndWait(); + txManager.stopAsync().awaitTerminated(); } @Test diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/file/TestFileLogging.java b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/file/TestFileLogging.java index eb4a831eb093..1ce373a30c2f 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/file/TestFileLogging.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/file/TestFileLogging.java @@ -66,7 +66,7 @@ public static void setUpContext() throws Exception { @AfterClass public static void cleanUp() throws Exception { - txManager.stopAndWait(); + txManager.stopAsync().awaitTerminated(); } @Test diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/system/CDAPLogAppenderTest.java b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/system/CDAPLogAppenderTest.java index c10d7574a8c2..109128a1242f 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/system/CDAPLogAppenderTest.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/system/CDAPLogAppenderTest.java @@ -110,14 +110,14 @@ protected void configure() { ); txManager = injector.getInstance(TransactionManager.class); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); StoreDefinition.LogFileMetaStore.create(injector.getInstance(StructuredTableAdmin.class)); } @AfterClass public static void cleanUp() { - txManager.stopAndWait(); + txManager.stopAsync().awaitTerminated(); } @Test diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/system/LogFileManagerTest.java b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/system/LogFileManagerTest.java index da165f4a050f..a709d1ac0a7e 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/system/LogFileManagerTest.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/system/LogFileManagerTest.java @@ -96,14 +96,14 @@ protected void configure() { ); txManager = injector.getInstance(TransactionManager.class); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); StoreDefinition.LogFileMetaStore.create(injector.getInstance(StructuredTableAdmin.class)); } @AfterClass public static void cleanUp() { - txManager.stopAndWait(); + txManager.stopAsync().awaitTerminated(); } @Test diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/clean/FileMetadataCleanerTest.java b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/clean/FileMetadataCleanerTest.java index 267a8bbeed97..92be6df7f60b 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/clean/FileMetadataCleanerTest.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/clean/FileMetadataCleanerTest.java @@ -108,13 +108,13 @@ protected void configure() { ); txManager = injector.getInstance(TransactionManager.class); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); StoreDefinition.LogFileMetaStore.create(injector.getInstance(StructuredTableAdmin.class)); } @AfterClass public static void cleanUp() { - txManager.stopAndWait(); + txManager.stopAsync().awaitTerminated(); } @Test diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/clean/LogCleanerTest.java b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/clean/LogCleanerTest.java index 6f968f2b295b..f5616618a653 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/clean/LogCleanerTest.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/clean/LogCleanerTest.java @@ -109,13 +109,13 @@ protected void configure() { ); txManager = injector.getInstance(TransactionManager.class); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); StoreDefinition.LogFileMetaStore.create(injector.getInstance(StructuredTableAdmin.class)); } @AfterClass public static void cleanUp() { - txManager.stopAndWait(); + txManager.stopAsync().awaitTerminated(); } @Test diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/framework/distributed/DistributedLogFrameworkTest.java b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/framework/distributed/DistributedLogFrameworkTest.java index 79a282ec35d9..aca1bc4ca70a 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/framework/distributed/DistributedLogFrameworkTest.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/framework/distributed/DistributedLogFrameworkTest.java @@ -112,19 +112,19 @@ public static void init() { @Before public void beforeTest() throws Exception { injector = createInjector(); - injector.getInstance(ZKClientService.class).startAndWait(); - injector.getInstance(KafkaClientService.class).startAndWait(); - injector.getInstance(BrokerService.class).startAndWait(); - injector.getInstance(TransactionManager.class).startAndWait(); + injector.getInstance(ZKClientService.class).startAsync().awaitRunning(); + injector.getInstance(KafkaClientService.class).startAsync().awaitRunning(); + injector.getInstance(BrokerService.class).startAsync().awaitRunning(); + injector.getInstance(TransactionManager.class).startAsync().awaitRunning(); StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class)); } @After public void afterTest() { - injector.getInstance(TransactionManager.class).stopAndWait(); - injector.getInstance(BrokerService.class).stopAndWait(); - injector.getInstance(KafkaClientService.class).stopAndWait(); - injector.getInstance(ZKClientService.class).stopAndWait(); + injector.getInstance(TransactionManager.class).stopAsync().awaitTerminated(); + injector.getInstance(BrokerService.class).stopAsync().awaitTerminated(); + injector.getInstance(KafkaClientService.class).stopAsync().awaitTerminated(); + injector.getInstance(ZKClientService.class).stopAsync().awaitTerminated(); injector = null; } @@ -133,7 +133,7 @@ public void testFramework() throws Exception { DistributedLogFramework framework = injector.getInstance(DistributedLogFramework.class); CConfiguration cConf = injector.getInstance(CConfiguration.class); - framework.startAndWait(); + framework.startAsync().awaitRunning(); // Send some logs to Kafka. LoggingContext context = new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), @@ -186,7 +186,7 @@ public void testFramework() throws Exception { } }, 10, TimeUnit.SECONDS, msgCount, TimeUnit.MILLISECONDS); - framework.stopAndWait(); + framework.stopAsync().awaitTerminated(); String kafkaTopic = cConf.get(Constants.Logging.KAFKA_TOPIC); // Check the checkpoint is persisted correctly. Since all messages are processed, diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/logbuffer/ConcurrentLogBufferWriterTest.java b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/logbuffer/ConcurrentLogBufferWriterTest.java index 9328e8654c33..d94bb6086bba 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/logbuffer/ConcurrentLogBufferWriterTest.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/logbuffer/ConcurrentLogBufferWriterTest.java @@ -81,7 +81,7 @@ public void testWrites() throws Exception { new LogProcessorPipelineContext(CConfiguration.create(), "test", loggerContext, NO_OP_METRICS_CONTEXT, 0), config, checkpointManager, 0); // start the pipeline - pipeline.startAndWait(); + pipeline.startAsync().awaitRunning(); ConcurrentLogBufferWriter writer = new ConcurrentLogBufferWriter(cConf, ImmutableList.of(pipeline), () -> { }); ImmutableList events = getLoggingEvents(); @@ -97,7 +97,7 @@ public void testWrites() throws Exception { // verify if the pipeline has processed the messages. Tasks.waitFor(5, () -> appender.getEvents().size(), 60, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); - pipeline.stopAndWait(); + pipeline.stopAsync().awaitTerminated(); loggerContext.stop(); } @@ -122,7 +122,7 @@ public void testConcurrentWrites() throws Exception { new LogProcessorPipelineContext(CConfiguration.create(), "test", loggerContext, NO_OP_METRICS_CONTEXT, 0), config, checkpointManager, 0); // start the pipeline - pipeline.startAndWait(); + pipeline.startAsync().awaitRunning(); ConcurrentLogBufferWriter writer = new ConcurrentLogBufferWriter(cConf, ImmutableList.of(pipeline), () -> { }); ImmutableList events = getLoggingEvents(); @@ -156,7 +156,7 @@ public void testConcurrentWrites() throws Exception { // verify if the pipeline has processed the messages. Tasks.waitFor(100, () -> appender.getEvents().size(), 60, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); - pipeline.stopAndWait(); + pipeline.stopAsync().awaitTerminated(); loggerContext.stop(); } diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/logbuffer/handler/LogBufferHandlerTest.java b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/logbuffer/handler/LogBufferHandlerTest.java index 78aa27e9775d..1f5e9dee9dac 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/logbuffer/handler/LogBufferHandlerTest.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/logbuffer/handler/LogBufferHandlerTest.java @@ -73,7 +73,7 @@ public void testHandler() throws Exception { "Test", MockAppender.class); LogBufferProcessorPipeline pipeline = getLogPipeline(loggerContext); - pipeline.startAndWait(); + pipeline.startAsync().awaitRunning(); ConcurrentLogBufferWriter writer = new ConcurrentLogBufferWriter(cConf, ImmutableList.of(pipeline), () -> { }); @@ -98,7 +98,7 @@ public void testHandler() throws Exception { remoteLogAppender.stop(); httpService.stop(); - pipeline.stopAndWait(); + pipeline.stopAsync().awaitTerminated(); loggerContext.stop(); } diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/logbuffer/recover/LogBufferRecoveryServiceTest.java b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/logbuffer/recover/LogBufferRecoveryServiceTest.java index 7e254b8b59a7..3d7cee3f57d7 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/logbuffer/recover/LogBufferRecoveryServiceTest.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/logbuffer/recover/LogBufferRecoveryServiceTest.java @@ -72,7 +72,7 @@ public void testLogBufferRecoveryService() throws Exception { config, checkpointManager, 0); // start the pipeline - pipeline.startAndWait(); + pipeline.startAsync().awaitRunning(); // write directly to log buffer LogBufferWriter writer = new LogBufferWriter(absolutePath, 250, () -> { }); @@ -85,12 +85,12 @@ public void testLogBufferRecoveryService() throws Exception { LogBufferRecoveryService service = new LogBufferRecoveryService(ImmutableList.of(pipeline), ImmutableList.of(checkpointManager), absolutePath, 2, new AtomicBoolean(true)); - service.startAndWait(); + service.startAsync().awaitRunning(); Tasks.waitFor(5, () -> appender.getEvents().size(), 120, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); - service.stopAndWait(); - pipeline.stopAndWait(); + service.stopAsync().awaitTerminated(); + pipeline.stopAsync().awaitTerminated(); loggerContext.stop(); } diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/pipeline/kafka/KafkaLogProcessorPipelineTest.java b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/pipeline/kafka/KafkaLogProcessorPipelineTest.java index 182c46c6fde7..9faf86d1f55c 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/pipeline/kafka/KafkaLogProcessorPipelineTest.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/pipeline/kafka/KafkaLogProcessorPipelineTest.java @@ -134,7 +134,7 @@ public void testBasicSort() throws Exception { checkpointManager, KAFKA_TESTER.getBrokerService(), config); - pipeline.startAndWait(); + pipeline.startAsync().awaitRunning(); // Publish some log messages to Kafka long now = System.currentTimeMillis(); @@ -176,7 +176,7 @@ public void testBasicSort() throws Exception { Assert.assertEquals("Large logger " + i, events.get(i).getMessage()); } - pipeline.stopAndWait(); + pipeline.stopAsync().awaitTerminated(); loggerContext.stop(); Assert.assertNull(appender.getEvents()); @@ -203,7 +203,7 @@ public void testRegularFlush() throws Exception { checkpointManager, KAFKA_TESTER.getBrokerService(), config); - pipeline.startAndWait(); + pipeline.startAsync().awaitRunning(); // Even when there is no event, the flush should still get called. Tasks.waitFor(5, appender::getFlushCount, 3, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); @@ -219,7 +219,7 @@ public void testRegularFlush() throws Exception { // Wait until getting all logs. Tasks.waitFor(3, () -> appender.getEvents().size(), 3, TimeUnit.SECONDS, 200, TimeUnit.MILLISECONDS); - pipeline.stopAndWait(); + pipeline.stopAsync().awaitTerminated(); // Should get at least 20 flush calls, since the checkpoint is every 2 seconds Assert.assertTrue(appender.getFlushCount() >= 20); @@ -229,7 +229,7 @@ public void testRegularFlush() throws Exception { public void testMetricsAppender() throws Exception { Injector injector = KAFKA_TESTER.getInjector(); MetricsCollectionService collectionService = injector.getInstance(MetricsCollectionService.class); - collectionService.startAndWait(); + collectionService.startAsync().awaitRunning(); LoggerContext loggerContext = new LocalAppenderContext(injector.getInstance(TransactionRunner.class), injector.getInstance(LocationFactory.class), injector.getInstance(MetricsCollectionService.class)); @@ -254,7 +254,7 @@ public void testMetricsAppender() throws Exception { checkpointManager, KAFKA_TESTER.getBrokerService(), config); - pipeline.startAndWait(); + pipeline.startAsync().awaitRunning(); // Publish some log messages to Kafka long now = System.currentTimeMillis(); @@ -337,9 +337,9 @@ public void testMetricsAppender() throws Exception { LoggingContextHelper.getMetricsTags(serviceLoggingContext), new ArrayList<>()), 3L); } finally { - pipeline.stopAndWait(); + pipeline.stopAsync().awaitTerminated(); loggerContext.stop(); - collectionService.stopAndWait(); + collectionService.stopAsync().awaitTerminated(); } } @@ -379,7 +379,7 @@ public void testMultiAppenders() throws Exception { checkpointManager, KAFKA_TESTER.getBrokerService(), config); - pipeline.startAndWait(); + pipeline.startAsync().awaitRunning(); // Publish some log messages to Kafka using a non-specific logger long now = System.currentTimeMillis(); @@ -439,7 +439,7 @@ public void testMultiAppenders() throws Exception { return Arrays.asList("TRACE", "DEBUG", "INFO", "WARN", "ERROR", "ERROR").equals(lines1); }, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); - pipeline.stopAndWait(); + pipeline.stopAsync().awaitTerminated(); loggerContext.stop(); } diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/pipeline/logbuffer/LogBufferProcessorPipelineTest.java b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/pipeline/logbuffer/LogBufferProcessorPipelineTest.java index ad28dae00ec2..8f9b85e57eed 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/pipeline/logbuffer/LogBufferProcessorPipelineTest.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/pipeline/logbuffer/LogBufferProcessorPipelineTest.java @@ -67,7 +67,7 @@ public void testSingleAppender() throws Exception { new LogProcessorPipelineContext(CConfiguration.create(), "test", loggerContext, NO_OP_METRICS_CONTEXT, 0), config, checkpointManager, 0); // start the pipeline - pipeline.startAndWait(); + pipeline.startAsync().awaitRunning(); // start thread to write to incomingEventQueue List events = getLoggingEvents(); @@ -95,7 +95,7 @@ public void testSingleAppender() throws Exception { // wait for pipeline to append all the logs to appender. The DEBUG message should get filtered out. Tasks.waitFor(200, () -> appender.getEvents().size(), 60, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); executorService.shutdown(); - pipeline.stopAndWait(); + pipeline.stopAsync().awaitTerminated(); loggerContext.stop(); } diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/plugins/RollingLocationLogAppenderTest.java b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/plugins/RollingLocationLogAppenderTest.java index 350709ec9cb9..e9a3b495cdc9 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/plugins/RollingLocationLogAppenderTest.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/plugins/RollingLocationLogAppenderTest.java @@ -110,12 +110,12 @@ protected void configure() { ); txManager = injector.getInstance(TransactionManager.class); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); } @AfterClass public static void cleanUp() throws Exception { - txManager.stopAndWait(); + txManager.stopAsync().awaitTerminated(); } @Test diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/read/FileMetadataTest.java b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/read/FileMetadataTest.java index 1003b4845e15..46b82d6b56e7 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/read/FileMetadataTest.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/read/FileMetadataTest.java @@ -99,13 +99,13 @@ protected void configure() { ); txManager = injector.getInstance(TransactionManager.class); - txManager.startAndWait(); + txManager.startAsync().awaitRunning(); StoreDefinition.LogFileMetaStore.create(injector.getInstance(StructuredTableAdmin.class)); } @AfterClass public static void cleanUp() { - txManager.stopAndWait(); + txManager.stopAsync().awaitTerminated(); } @Test diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/metrics/MetricsTestBase.java b/cdap-watchdog/src/test/java/io/cdap/cdap/metrics/MetricsTestBase.java index c3d6f4bc8a6a..0e18f1d72918 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/metrics/MetricsTestBase.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/metrics/MetricsTestBase.java @@ -82,7 +82,7 @@ public void init() throws IOException, UnsupportedTypeException { injector = Guice.createInjector(getModules()); messagingService = injector.getInstance(MessagingService.class); if (messagingService instanceof Service) { - ((Service) messagingService).startAndWait(); + ((Service) messagingService).startAsync().awaitRunning(); } metricValueType = TypeToken.of(MetricValues.class); schema = new ReflectionSchemaGenerator().generate(metricValueType.getType()); @@ -93,7 +93,7 @@ public void init() throws IOException, UnsupportedTypeException { @After public void stop() { if (messagingService instanceof Service) { - ((Service) messagingService).stopAndWait(); + ((Service) messagingService).stopAsync().awaitTerminated(); } } diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/metrics/collect/AggregatedMetricsCollectionServiceTest.java b/cdap-watchdog/src/test/java/io/cdap/cdap/metrics/collect/AggregatedMetricsCollectionServiceTest.java index 6066bbfb3961..aa1783ae48f8 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/metrics/collect/AggregatedMetricsCollectionServiceTest.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/metrics/collect/AggregatedMetricsCollectionServiceTest.java @@ -74,7 +74,7 @@ protected void publish(Iterator metrics) { } }; - service.startAndWait(); + service.startAsync().awaitRunning(); // non-empty tags. final Map baseTags = ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, NAMESPACE, @@ -139,7 +139,7 @@ protected void publish(Iterator metrics) { metricsContext.gauge(GAUGE_METRIC, 0); verifyCounterMetricsValue(published, ImmutableMap.of(6, ImmutableMap.of(GAUGE_METRIC, 0L))); } finally { - service.stopAndWait(); + service.stopAsync().awaitTerminated(); } } @@ -211,7 +211,7 @@ protected void publish(Iterator metrics) { } }; - service.startAndWait(); + service.startAsync().awaitRunning(); // non-empty tags. final Map baseTags = ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, NAMESPACE, @@ -245,7 +245,7 @@ protected void publish(Iterator metrics) { verifyDistribtionMetricValues(published, 2, 4); } finally { - service.stopAndWait(); + service.stopAsync().awaitTerminated(); } } diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/metrics/collect/MessagingMetricsCollectionServiceTest.java b/cdap-watchdog/src/test/java/io/cdap/cdap/metrics/collect/MessagingMetricsCollectionServiceTest.java index 36322dd160d5..52abf3070fb5 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/metrics/collect/MessagingMetricsCollectionServiceTest.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/metrics/collect/MessagingMetricsCollectionServiceTest.java @@ -62,14 +62,14 @@ public void testMessagingPublish() throws TopicNotFoundException { MetricsCollectionService collectionService = new MessagingMetricsCollectionService(CConfiguration.create(), messagingService, recordWriter); - collectionService.startAndWait(); + collectionService.startAsync().awaitRunning(); // publish metrics for different context for (int i = 1; i <= 3; i++) { collectionService.getContext(ImmutableMap.of("tag", "" + i)).increment("processed", i); } - collectionService.stopAndWait(); + collectionService.stopAsync().awaitTerminated(); // Table expected = HashBasedTable.create(); diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/metrics/process/MessagingMetricsProcessorManagerServiceTest.java b/cdap-watchdog/src/test/java/io/cdap/cdap/metrics/process/MessagingMetricsProcessorManagerServiceTest.java index 99951102a14a..b47886b50c5a 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/metrics/process/MessagingMetricsProcessorManagerServiceTest.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/metrics/process/MessagingMetricsProcessorManagerServiceTest.java @@ -68,10 +68,10 @@ public class MessagingMetricsProcessorManagerServiceTest extends MetricsProcesso @Test public void persistMetricsTests() throws Exception { - injector.getInstance(TransactionManager.class).startAndWait(); + injector.getInstance(TransactionManager.class).startAsync().awaitRunning(); StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class)); - injector.getInstance(DatasetOpExecutorService.class).startAndWait(); - injector.getInstance(DatasetService.class).startAndWait(); + injector.getInstance(DatasetOpExecutorService.class).startAsync().awaitRunning(); + injector.getInstance(DatasetService.class).startAsync().awaitRunning(); Set partitions = IntStream.range(0, cConf.getInt(Constants.Metrics.MESSAGING_TOPIC_NUM)) .boxed().collect(Collectors.toSet()); @@ -106,7 +106,7 @@ public void persistMetricsTests() throws Exception { injector.getInstance(DatumReaderFactory.class), metricStore, injector.getInstance(MetricsWriterProvider.class), partitions, new NoopMetricsContext(), 50, 0); - messagingMetricsProcessorManagerService.startAndWait(); + messagingMetricsProcessorManagerService.startAsync().awaitRunning(); // Wait for the 1 aggregated counter metric (with value 50) and 50 gauge metrics to be stored in the metricStore Tasks.waitFor(51, () -> metricStore.getAllCounterAndGaugeMetrics().size(), 15, @@ -139,7 +139,7 @@ public void persistMetricsTests() throws Exception { metricStore.deleteAll(); expected.clear(); // Stop messagingMetricsProcessorManagerService - messagingMetricsProcessorManagerService.stopAndWait(); + messagingMetricsProcessorManagerService.stopAsync().awaitTerminated(); } } diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/metrics/process/MetricsAdminSubscriberServiceTest.java b/cdap-watchdog/src/test/java/io/cdap/cdap/metrics/process/MetricsAdminSubscriberServiceTest.java index d699e4d73475..0c9a9a5c1d09 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/metrics/process/MetricsAdminSubscriberServiceTest.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/metrics/process/MetricsAdminSubscriberServiceTest.java @@ -116,25 +116,25 @@ protected void configure() { metricsQueryService = injector.getInstance(MetricsQueryService.class); if (messagingService instanceof Service) { - ((Service) messagingService).startAndWait(); + ((Service) messagingService).startAsync().awaitRunning(); } - metricsCollectionService.startAndWait(); - metricsQueryService.startAndWait(); + metricsCollectionService.startAsync().awaitRunning(); + metricsQueryService.startAsync().awaitRunning(); } @AfterClass public static void finish() { - metricsQueryService.stopAndWait(); - metricsCollectionService.stopAndWait(); + metricsQueryService.stopAsync().awaitTerminated(); + metricsCollectionService.stopAsync().awaitTerminated(); if (messagingService instanceof Service) { - ((Service) messagingService).stopAndWait(); + ((Service) messagingService).stopAsync().awaitTerminated(); } } @Test public void test() throws Exception { MetricsAdminSubscriberService adminService = injector.getInstance(MetricsAdminSubscriberService.class); - adminService.startAndWait(); + adminService.startAsync().awaitRunning(); // publish a metrics MetricsContext metricsContext = metricsCollectionService.getContext( @@ -222,6 +222,6 @@ public void test() throws Exception { return !foundInc && !foundGauge; }, 1000, TimeUnit.SECONDS, 1, TimeUnit.SECONDS); - adminService.stopAndWait(); + adminService.stopAsync().awaitTerminated(); } } diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/metrics/process/MetricsProcessorServiceTest.java b/cdap-watchdog/src/test/java/io/cdap/cdap/metrics/process/MetricsProcessorServiceTest.java index 6168b40cd6ed..4f5c657783ed 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/metrics/process/MetricsProcessorServiceTest.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/metrics/process/MetricsProcessorServiceTest.java @@ -59,10 +59,10 @@ public class MetricsProcessorServiceTest extends MetricsProcessorServiceTestBase @Test public void testMetricsProcessor() throws Exception { - injector.getInstance(TransactionManager.class).startAndWait(); + injector.getInstance(TransactionManager.class).startAsync().awaitRunning(); StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class)); - injector.getInstance(DatasetOpExecutorService.class).startAndWait(); - injector.getInstance(DatasetService.class).startAndWait(); + injector.getInstance(DatasetOpExecutorService.class).startAsync().awaitRunning(); + injector.getInstance(DatasetService.class).startAsync().awaitRunning(); final MetricStore metricStore = injector.getInstance(MetricStore.class); @@ -81,7 +81,7 @@ public void testMetricsProcessor() throws Exception { injector.getInstance(DatumReaderFactory.class), metricStore, injector.getInstance(MetricsWriterProvider.class), partitions, new NoopMetricsContext(), 50, 0); - messagingMetricsProcessorManagerService.startAndWait(); + messagingMetricsProcessorManagerService.startAsync().awaitRunning(); long startTime = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()); // Publish metrics with messaging service and record expected metrics @@ -91,7 +91,7 @@ public void testMetricsProcessor() throws Exception { Thread.sleep(500); // Stop and restart messagingMetricsProcessorManagerService - messagingMetricsProcessorManagerService.stopAndWait(); + messagingMetricsProcessorManagerService.stopAsync().awaitTerminated(); // Intentionally set queue size to a large value, so that MessagingMetricsProcessorManagerService // internally only persists metrics during terminating. messagingMetricsProcessorManagerService = @@ -100,7 +100,7 @@ public void testMetricsProcessor() throws Exception { injector.getInstance(DatumReaderFactory.class), metricStore, injector.getInstance(MetricsWriterProvider.class), partitions, new NoopMetricsContext(), 50, 0); - messagingMetricsProcessorManagerService.startAndWait(); + messagingMetricsProcessorManagerService.startAsync().awaitRunning(); // Publish metrics after MessagingMetricsProcessorManagerService restarts and record expected metrics for (int i = 20; i < 30; i++) { @@ -138,7 +138,7 @@ public Boolean call() throws Exception { } // Stop services and servers - messagingMetricsProcessorManagerService.stopAndWait(); + messagingMetricsProcessorManagerService.stopAsync().awaitTerminated(); // Delete all metrics metricStore.deleteAll(); } diff --git a/pom.xml b/pom.xml index 64c19fa20c92..1ba9d4358a9b 100644 --- a/pom.xml +++ b/pom.xml @@ -104,7 +104,7 @@ ETLRealtime 2.0.0-M2 - 7.1 + 9.6 1.9.40 1.11.4 1.70 @@ -123,7 +123,7 @@ 2.0.0 0.1.0 2.3.1 - 13.0.1 + 32.0.0-jre 4.0 3.3.6 2.2.4 @@ -148,7 +148,7 @@ 3.0.8.Final 2.0 - 2.12.15 + 2.12.18 3.1.0 1.7.15 1.1.1.7 @@ -158,7 +158,7 @@ 0.15.0-incubating 0.8.4 0.9.3 - 1.4.0 + 1.5.0-SNAPSHOT 2.3.6 3.4.5 1.3.1 @@ -180,7 +180,7 @@ true - + --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.management/java.lang.management=ALL-UNNAMED -Djava.locale.providers=COMPAT,CLDR @@ -444,11 +444,21 @@ logback-classic ${logback.version} + + org.ow2.asm + asm + ${asm.version} + org.ow2.asm asm-commons ${asm.version} + + org.ow2.asm + asm-tree + ${asm.version} + org.apache.kafka kafka_2.12 @@ -485,6 +495,16 @@ scala-library ${scala2.12.version} + + org.scala-lang + scala-compiler + ${scala2.12.version} + + + org.scala-lang + scala-reflect + ${scala2.12.version} + org.apache.kafka kafka-clients @@ -1508,7 +1528,7 @@ maven-surefire-plugin 3.0.0-M6 - @{argLine} -Xmx3500m -Djava.awt.headless=true -XX:+UseG1GC -XX:OnOutOfMemoryError="kill -9 %p" -XX:+HeapDumpOnOutOfMemoryError + @{argLine} -Xmx3500m -Djava.awt.headless=true -XX:+UseG1GC -XX:OnOutOfMemoryError="kill -9 %p" -XX:+HeapDumpOnOutOfMemoryError --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.management/java.lang.management=ALL-UNNAMED ${surefire.redirectTestOutputToFile} false 3 @@ -1579,7 +1599,7 @@ plugins/** **/LICENSES/** **/VERSION - **/*.patch + **/*.patch **/logrotate.d/** **/limits.d/** @@ -1596,6 +1616,8 @@ third-party-licenses/** .gitpod.yml .gitpod.Dockerfile + .mvn/** + *thread_dump.txt @@ -1696,6 +1718,11 @@ true + + org.apache.maven.plugins + maven-jar-plugin + 3.5.0 + @@ -1836,7 +1863,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 2.9.1 + 3.6.3 cdap-distributions/src/javadoc/stylesheet.css @@ -1845,11 +1872,14 @@ ${project.name} ${project.version} https://cdapio.github.io/docs-site/hosted-javadocs/jsr305-${findbugs.jsr305.version}-javadoc/ - http://avro.apache.org/docs/${avro.version}/api/java/ - http://docs.oracle.com/javaee/${jee.version}/api/ - http://hadoop.apache.org/docs/r${hadoop.version}/api/ - http://twill.incubator.apache.org/apidocs-${twill.version}/ + https://avro.apache.org/docs/${avro.version}/api/java/ + https://javaee.github.io/javaee-spec/javadocs/ + https://hadoop.apache.org/docs/r${hadoop.version}/api/ + https://twill.apache.org/apidocs-${twill.version}/ + none + false + false true @@ -1878,7 +1908,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 2.9.1 + 3.6.3 @@ -1915,10 +1945,9 @@ org.apache.maven.plugins maven-jar-plugin - 2.4 - jar + default-jar prepare-package ${stage.lib.dir} @@ -2645,9 +2674,9 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.1.0 + 3.6.3 - -Xdoclint:none + none false @@ -2698,9 +2727,9 @@ org.apache.maven.plugins maven-javadoc-plugin - 2.9.1 + 3.6.3 - -Xdoclint:none + none false cdap-distributions/src/javadoc/stylesheet.css @@ -2709,11 +2738,12 @@ ${project.name} ${project.version} https://cdapio.github.io/docs-site/hosted-javadocs/jsr305-${findbugs.jsr305.version}-javadoc/ - http://avro.apache.org/docs/${avro.version}/api/java/ - http://docs.oracle.com/javaee/${jee.version}/api/ - http://hadoop.apache.org/docs/r${hadoop.version}/api/ - http://twill.incubator.apache.org/apidocs-${twill.version}/ + https://avro.apache.org/docs/${avro.version}/api/java/ + https://javaee.github.io/javaee-spec/javadocs/ + https://hadoop.apache.org/docs/r${hadoop.version}/api/ + https://twill.apache.org/apidocs-${twill.version}/ + false true diff --git a/suppressions.xml b/suppressions.xml index fb0862f157ff..699c57c59c35 100644 --- a/suppressions.xml +++ b/suppressions.xml @@ -41,7 +41,9 @@ - + + +