diff --git a/DocumentsFromSnapshotMigration/docker/entrypoint.sh b/DocumentsFromSnapshotMigration/docker/entrypoint.sh index a16202fa4..7deb44da9 100755 --- a/DocumentsFromSnapshotMigration/docker/entrypoint.sh +++ b/DocumentsFromSnapshotMigration/docker/entrypoint.sh @@ -18,7 +18,7 @@ echo "RFS_TARGET_PASSWORD_ARN: $RFS_TARGET_PASSWORD_ARN" if [[ $RFS_COMMAND != *"--target-username"* ]]; then if [[ -n "$RFS_TARGET_USER" ]]; then echo "Using username from ENV variable RFS_TARGET_USER. Updating RFS Command with username." - RFS_COMMAND="$RFS_COMMAND --target-username $RFS_TARGET_USER" + RFS_COMMAND="$RFS_COMMAND --target-username \"$RFS_TARGET_USER\"" fi fi @@ -39,9 +39,9 @@ if [[ $RFS_COMMAND != *"--target-password"* ]]; then # Append the username/password to the RFS Command if have an updated password if [[ -n "$PASSWORD_TO_USE" ]]; then echo "Updating RFS Command with password." - RFS_COMMAND="$RFS_COMMAND --target-password $PASSWORD_TO_USE" + RFS_COMMAND="$RFS_COMMAND --target-password \"$PASSWORD_TO_USE\"" fi fi echo "Executing RFS Command" -eval $RFS_COMMAND \ No newline at end of file +eval $RFS_COMMAND diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java index ab5287064..4a82315c0 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java @@ -33,6 +33,7 @@ import org.apache.kafka.common.config.SaslConfigs; import org.opensearch.common.settings.Settings; +import org.opensearch.migrations.jcommander.NoSplitter; import org.opensearch.migrations.tracing.ActiveContextTracker; import org.opensearch.migrations.tracing.ActiveContextTrackerByActivityType; import org.opensearch.migrations.tracing.CompositeContextTracker; @@ -158,12 +159,14 @@ public static class Parameters { public String otelCollectorEndpoint; @Parameter(required = false, names = "--setHeader", + splitter = NoSplitter.class, arity = 2, description = "[header-name header-value] Set an HTTP header (first argument) with to the specified value" + " (second argument). Any existing headers with that name will be removed.") public List headerOverrides = new ArrayList<>(); @Parameter(required = false, names = "--suppressCaptureForHeaderMatch", + splitter = NoSplitter.class, arity = 2, description = "The header name (which will be interpreted in a case-insensitive manner) and a regex " + "pattern. When the incoming request has a header that matches the regex, it will be passed " diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java index 45f8f732d..73b88ad58 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java @@ -15,6 +15,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import org.opensearch.migrations.jcommander.NoSplitter; import org.opensearch.migrations.replay.tracing.RootReplayerContext; import org.opensearch.migrations.replay.traffic.source.TrafficStreamLimiter; import org.opensearch.migrations.replay.util.ActiveContextMonitor; @@ -117,6 +118,7 @@ public static class Parameters { @Parameter( required = false, names = { AWS_AUTH_HEADER_USER_AND_SECRET_ARG }, + splitter = NoSplitter.class, arity = 2, description = " pair to specify " + "\"authorization\" header value for each request. " diff --git a/coreUtilities/build.gradle b/coreUtilities/build.gradle index 7b84ff3cb..0a16ed15e 100644 --- a/coreUtilities/build.gradle +++ b/coreUtilities/build.gradle @@ -41,6 +41,9 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core' implementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j2-impl' + // JCommander + compileOnly group: 'org.jcommander', name: 'jcommander' + // OpenTelemetry core api group: 'io.opentelemetry', name: 'opentelemetry-api' api group: 'io.opentelemetry', name: 'opentelemetry-sdk' diff --git a/coreUtilities/src/main/java/org/opensearch/migrations/jcommander/NoSplitter.java b/coreUtilities/src/main/java/org/opensearch/migrations/jcommander/NoSplitter.java new file mode 100644 index 000000000..404d29af4 --- /dev/null +++ b/coreUtilities/src/main/java/org/opensearch/migrations/jcommander/NoSplitter.java @@ -0,0 +1,12 @@ +package org.opensearch.migrations.jcommander; + +import java.util.List; + +import com.beust.jcommander.converters.IParameterSplitter; + +public class NoSplitter implements IParameterSplitter { + @Override + public List split(String value) { + return List.of(value); + } +} diff --git a/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts b/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts index 0da81c177..b24275166 100644 --- a/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts +++ b/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts @@ -4,87 +4,66 @@ import {CpuArchitecture} from "aws-cdk-lib/aws-ecs"; import {RemovalPolicy, Stack} from "aws-cdk-lib"; import { IStringParameter, StringParameter } from "aws-cdk-lib/aws-ssm"; import * as forge from 'node-forge'; -import * as yargs from 'yargs'; import { ClusterYaml } from "./migration-services-yaml"; +export function getTargetPasswordAccessPolicy(targetPasswordSecretArn: string): PolicyStatement { + return new PolicyStatement({ + effect: Effect.ALLOW, + resources: [targetPasswordSecretArn], + actions: [ + "secretsmanager:GetSecretValue" + ] + }) +} -// parseAndMergeArgs, see @common-utilities.test.ts for an example of different cases -export function parseAndMergeArgs(baseCommand: string, extraArgs?: string): string { - if (!extraArgs) { - return baseCommand; +export function appendArgIfNotInExtraArgs( + baseCommand: string, + extraArgsDict: Record, + arg: string, + value: string | null = null, +): string { + if (extraArgsDict[arg] === undefined) { + // If not present, append the argument and value (only append value if it exists) + baseCommand = value !== null ? baseCommand.concat(" ", arg, " ", value) : baseCommand.concat(" ", arg); } + return baseCommand; +} - // Extract command prefix - const commandPrefix = baseCommand.substring(0, baseCommand.indexOf('--')).trim(); - const baseArgs = baseCommand.substring(baseCommand.indexOf('--')); +export function parseArgsToDict(argString: string | undefined): Record { + const args: Record = {}; + if (argString === undefined) { + return args; + } + // Split based on '--' at the start of the string or preceded by whitespace, use non-capturing groups to include -- in parts + const parts = argString.split(/(?=\s--|^--)/).filter(Boolean); - // Parse base command - const baseYargsConfig = { - parserConfiguration: { - 'camel-case-expansion': false, - 'boolean-negation': false, - } - }; + parts.forEach(part => { + const trimmedPart = part.trim(); + if (trimmedPart.length === 0) return; // Skip empty parts - const baseArgv = yargs(baseArgs) - .parserConfiguration(baseYargsConfig.parserConfiguration) - .parse(); + // Use a regular expression to find the first whitespace character + const firstWhitespaceMatch = trimmedPart.match(/\s/); + const firstWhitespaceIndex = firstWhitespaceMatch?.index; - // Parse extra args if provided - const extraYargsConfig = { - parserConfiguration: { - 'camel-case-expansion': false, - 'boolean-negation': true, - } - }; - - const extraArgv = extraArgs - ? yargs(extraArgs.split(' ')) - .parserConfiguration(extraYargsConfig.parserConfiguration) - .parse() - : {}; - - // Merge arguments - const mergedArgv: { [key: string]: unknown } = { ...baseArgv }; - for (const [key, value] of Object.entries(extraArgv)) { - if (key !== '_' && key !== '$0') { - if (!value && - typeof value === 'boolean' && - ( - typeof (baseArgv as any)[key] === 'boolean' || - (typeof (baseArgv as any)[`no-${key}`] != 'boolean' && typeof (baseArgv as any)[`no-${key}`]) - ) - ) { - delete mergedArgv[key]; + const key = firstWhitespaceIndex === undefined ? trimmedPart : trimmedPart.slice(0, firstWhitespaceIndex).trim(); + const value = firstWhitespaceIndex === undefined ? '' : trimmedPart.slice(firstWhitespaceIndex + 1).trim(); + + // Validate the key starts with -- followed by a non-whitespace characters + if (/^--\S+/.test(key)) { + if (args[key] !== undefined) { + args[key].push(value); } else { - mergedArgv[key] = value; + args[key] = [value]; } + } else { + throw new Error(`Invalid argument key: '${key}'. Argument keys must start with '--' and contain no spaces.`); } + }); + if (argString.trim() && !args) { + throw new Error(`Unable to parse args provided: '${argString}'`); } - // Reconstruct command - const mergedArgs = Object.entries(mergedArgv) - .filter(([key]) => key !== '_' && key !== '$0') - .map(([key, value]) => { - if (typeof value === 'boolean') { - return value ? `--${key}` : `--no-${key}`; - } - return `--${key} ${value}`; - }) - .join(' '); - - let fullCommand = `${commandPrefix} ${mergedArgs}`.trim() - return fullCommand; -} - -export function getTargetPasswordAccessPolicy(targetPasswordSecretArn: string): PolicyStatement { - return new PolicyStatement({ - effect: Effect.ALLOW, - resources: [targetPasswordSecretArn], - actions: [ - "secretsmanager:GetSecretValue" - ] - }) + return args; } export function createOpenSearchIAMAccessPolicy(partition: string, region: string, accountId: string): PolicyStatement { diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-es-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-es-stack.ts index 91d9e1379..f2e4793dc 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-es-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-es-stack.ts @@ -8,8 +8,7 @@ import {StreamingSourceType} from "../streaming-source-type"; import { MigrationSSMParameter, createMSKProducerIAMPolicies, - getMigrationStringParameterValue, - parseAndMergeArgs + getMigrationStringParameterValue, parseArgsToDict, appendArgIfNotInExtraArgs, } from "../common-utilities"; import {OtelCollectorSidecar} from "./migration-otel-collector-sidecar"; @@ -63,11 +62,22 @@ export class CaptureProxyESStack extends MigrationServiceCore { ...props, parameter: MigrationSSMParameter.KAFKA_BROKERS, }); - let command = `/usr/local/bin/docker-entrypoint.sh eswrapper & /runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.CaptureProxy --destinationUri https://localhost:19200 --insecureDestination --listenPort 9200 --sslConfigFile /usr/share/elasticsearch/config/proxy_tls.yml` - command = props.streamingSourceType !== StreamingSourceType.DISABLED ? command.concat(` --kafkaConnection ${brokerEndpoints}`) : command - command = props.streamingSourceType === StreamingSourceType.AWS_MSK ? command.concat(" --enableMSKAuth") : command - command = props.otelCollectorEnabled ? command.concat(` --otelCollectorEndpoint ${OtelCollectorSidecar.getOtelLocalhostEndpoint()}`) : command - command = parseAndMergeArgs(command, props.extraArgs); + + let command = "/usr/local/bin/docker-entrypoint.sh eswrapper & /runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.CaptureProxy" + const extraArgsDict = parseArgsToDict(props.extraArgs) + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--destinationUri", "https://localhost:19200") + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--insecureDestination", "https://localhost:19200") + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--sslConfigFile", "/usr/share/elasticsearch/config/proxy_tls.yml") + if (props.streamingSourceType !== StreamingSourceType.DISABLED) { + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--kafkaConnection", brokerEndpoints) + } + if (props.streamingSourceType === StreamingSourceType.AWS_MSK) { + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--enableMSKAuth") + } + if (props.otelCollectorEnabled) { + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--otelCollectorEndpoint", OtelCollectorSidecar.getOtelLocalhostEndpoint()) + } + command = props.extraArgs ? command.concat(` ${props.extraArgs}`) : command this.createService({ serviceName: "capture-proxy-es", diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-stack.ts index 49e57a212..abd31794a 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-stack.ts @@ -9,8 +9,7 @@ import { MigrationSSMParameter, createMSKProducerIAMPolicies, getCustomStringParameterValue, - getMigrationStringParameterValue, - parseAndMergeArgs + getMigrationStringParameterValue, parseArgsToDict, appendArgIfNotInExtraArgs, } from "../common-utilities"; import {OtelCollectorSidecar} from "./migration-otel-collector-sidecar"; @@ -122,11 +121,23 @@ export class CaptureProxyStack extends MigrationServiceCore { const destinationEndpoint = getDestinationEndpoint(this, props.destinationConfig, props); - let command = `/runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.CaptureProxy --destinationUri ${destinationEndpoint} --insecureDestination --listenPort 9200 --sslConfigFile /usr/share/elasticsearch/config/proxy_tls.yml` - command = props.streamingSourceType !== StreamingSourceType.DISABLED ? command.concat(` --kafkaConnection ${brokerEndpoints}`) : command - command = props.streamingSourceType === StreamingSourceType.AWS_MSK ? command.concat(" --enableMSKAuth") : command - command = props.otelCollectorEnabled ? command.concat(` --otelCollectorEndpoint ${OtelCollectorSidecar.getOtelLocalhostEndpoint()}`) : command - command = parseAndMergeArgs(command, props.extraArgs); + let command = "/runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.CaptureProxy" + + const extraArgsDict = parseArgsToDict(props.extraArgs) + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--destinationUri", destinationEndpoint) + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--insecureDestination") + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--listenPort", "9200") + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--sslConfigFile", "/usr/share/elasticsearch/config/proxy_tls.yml") + if (props.streamingSourceType !== StreamingSourceType.DISABLED) { + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--kafkaConnection", brokerEndpoints) + } + if (props.streamingSourceType === StreamingSourceType.AWS_MSK) { + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--enableMSKAuth") + } + if (props.otelCollectorEnabled) { + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--otelCollectorEndpoint", OtelCollectorSidecar.getOtelLocalhostEndpoint()) + } + command = props.extraArgs ? command.concat(` ${props.extraArgs}`) : command this.createService({ serviceName: serviceName, diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/reindex-from-snapshot-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/reindex-from-snapshot-stack.ts index 8cb5608ad..cc4d693d2 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/reindex-from-snapshot-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/reindex-from-snapshot-stack.ts @@ -11,8 +11,7 @@ import { createOpenSearchServerlessIAMAccessPolicy, getTargetPasswordAccessPolicy, getMigrationStringParameterValue, - parseAndMergeArgs, - ClusterAuth + ClusterAuth, parseArgsToDict, appendArgIfNotInExtraArgs } from "../common-utilities"; import { RFSBackfillYaml, SnapshotYaml } from "../migration-services-yaml"; import { OtelCollectorSidecar } from "./migration-otel-collector-sidecar"; @@ -69,20 +68,40 @@ export class ReindexFromSnapshotStack extends MigrationServiceCore { parameter: MigrationSSMParameter.OS_CLUSTER_ENDPOINT, }); const s3Uri = `s3://migration-artifacts-${this.account}-${props.stage}-${this.region}/rfs-snapshot-repo`; - let rfsCommand = `/rfs-app/runJavaWithClasspath.sh org.opensearch.migrations.RfsMigrateDocuments --s3-local-dir /tmp/s3_files --s3-repo-uri ${s3Uri} --s3-region ${this.region} --snapshot-name rfs-snapshot --lucene-dir '/lucene' --target-host ${osClusterEndpoint}` - rfsCommand = props.clusterAuthDetails.sigv4 ? rfsCommand.concat(`--target-aws-service-signing-name ${props.clusterAuthDetails.sigv4.serviceSigningName} --target-aws-region ${props.clusterAuthDetails.sigv4.region}`) : rfsCommand - rfsCommand = props.otelCollectorEnabled ? rfsCommand.concat(` --otel-collector-endpoint ${OtelCollectorSidecar.getOtelLocalhostEndpoint()}`) : rfsCommand - rfsCommand = props.sourceClusterVersion ? rfsCommand.concat(` --source-version ${props.sourceClusterVersion}`) : rfsCommand - rfsCommand = parseAndMergeArgs(rfsCommand, props.extraArgs); + let command = "/rfs-app/runJavaWithClasspath.sh org.opensearch.migrations.RfsMigrateDocuments" + const extraArgsDict = parseArgsToDict(props.extraArgs) + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--s3-local-dir", "/tmp/s3_files") + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--s3-repo-uri", `"${s3Uri}"`) + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--s3-region", this.region) + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--snapshot-name", "rfs-snapshot") + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--lucene-dir", "/lucene") + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--target-host", osClusterEndpoint) + if (props.clusterAuthDetails.sigv4) { + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--target-aws-service-signing-name", props.clusterAuthDetails.sigv4.serviceSigningName) + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--target-aws-region", props.clusterAuthDetails.sigv4.region) + } + if (props.otelCollectorEnabled) { + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--otel-collector-endpoint", OtelCollectorSidecar.getOtelLocalhostEndpoint()) + } + if (props.sourceClusterVersion) { + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--source-version", `"${props.sourceClusterVersion}"`) + } let targetUser = ""; let targetPassword = ""; let targetPasswordArn = ""; if (props.clusterAuthDetails.basicAuth) { - targetUser = props.clusterAuthDetails.basicAuth.username - targetPassword = props.clusterAuthDetails.basicAuth.password ?? "" - targetPasswordArn = props.clusterAuthDetails.basicAuth.password_from_secret_arn ?? "" - }; + // Only set user or password if not overridden in extraArgs + if (extraArgsDict["--target-username"] === undefined) { + targetUser = props.clusterAuthDetails.basicAuth.username + } + if (extraArgsDict["--target-password"] === undefined) { + targetPassword = props.clusterAuthDetails.basicAuth.password ?? "" + targetPasswordArn = props.clusterAuthDetails.basicAuth.password_from_secret_arn ?? "" + } + } + command = props.extraArgs ? command.concat(` ${props.extraArgs}`) : command + const sharedLogFileSystem = new SharedLogFileSystem(this, props.stage, props.defaultDeployId); const openSearchPolicy = createOpenSearchIAMAccessPolicy(this.partition, this.region, this.account); const openSearchServerlessPolicy = createOpenSearchServerlessIAMAccessPolicy(this.partition, this.region, this.account); @@ -108,7 +127,7 @@ export class ReindexFromSnapshotStack extends MigrationServiceCore { taskMemoryLimitMiB: 4096, ephemeralStorageGiB: 200, environment: { - "RFS_COMMAND": rfsCommand, + "RFS_COMMAND": command, "RFS_TARGET_USER": targetUser, "RFS_TARGET_PASSWORD": targetPassword, "RFS_TARGET_PASSWORD_ARN": targetPasswordArn, diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-replayer-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-replayer-stack.ts index 8dcd964d2..aa363fe0b 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-replayer-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-replayer-stack.ts @@ -11,7 +11,7 @@ import { createMSKConsumerIAMPolicies, createOpenSearchIAMAccessPolicy, createOpenSearchServerlessIAMAccessPolicy, - getMigrationStringParameterValue, parseAndMergeArgs + getMigrationStringParameterValue, appendArgIfNotInExtraArgs, parseArgsToDict } from "../common-utilities"; import {StreamingSourceType} from "../streaming-source-type"; import { Duration } from "aws-cdk-lib"; @@ -80,21 +80,37 @@ export class TrafficReplayerStack extends MigrationServiceCore { }); const groupId = props.customKafkaGroupId ? props.customKafkaGroupId : `logging-group-${deployId}` - let replayerCommand = `/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer ${osClusterEndpoint} --insecure --kafka-traffic-brokers ${brokerEndpoints} --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id ${groupId}` + let command = `/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer ${osClusterEndpoint}` + const extraArgsDict = parseArgsToDict(props.extraArgs) + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--insecure") + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--kafka-traffic-brokers", brokerEndpoints) + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--kafka-traffic-topic", "logging-traffic-topic") + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--kafka-traffic-group-id", groupId) + if (props.clusterAuthDetails.basicAuth) { - replayerCommand = replayerCommand.concat(` --auth-header-user-and-secret "${props.clusterAuthDetails.basicAuth.username} ${props.clusterAuthDetails.basicAuth.password_from_secret_arn}"`) + const bashSafeUserAndSecret = `"${props.clusterAuthDetails.basicAuth.username}" "${props.clusterAuthDetails.basicAuth.password_from_secret_arn}"` + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--auth-header-user-and-secret", bashSafeUserAndSecret) + } + if (props.streamingSourceType === StreamingSourceType.AWS_MSK) { + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--kafka-traffic-enable-msk-auth") + } + if (props.userAgentSuffix) { + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--user-agent", `"${props.userAgentSuffix}"`) + } + if (props.clusterAuthDetails.sigv4) { + const sigv4AuthHeaderServiceRegion = `${props.clusterAuthDetails.sigv4.serviceSigningName},${props.clusterAuthDetails.sigv4.region}` + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--sigv4-auth-header-service-region", sigv4AuthHeaderServiceRegion) + } + if (props.otelCollectorEnabled) { + command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--otelCollectorEndpoint", OtelCollectorSidecar.getOtelLocalhostEndpoint()) } - replayerCommand = props.streamingSourceType === StreamingSourceType.AWS_MSK ? replayerCommand.concat(" --kafka-traffic-enable-msk-auth") : replayerCommand - replayerCommand = props.userAgentSuffix ? replayerCommand.concat(` --user-agent ${props.userAgentSuffix}`) : replayerCommand - replayerCommand = props.clusterAuthDetails.sigv4 ? replayerCommand.concat(` --sigv4-auth-header-service-region ${props.clusterAuthDetails.sigv4.serviceSigningName},${props.clusterAuthDetails.sigv4.region}`) : replayerCommand - replayerCommand = props.otelCollectorEnabled ? replayerCommand.concat(` --otelCollectorEndpoint ${OtelCollectorSidecar.getOtelLocalhostEndpoint()}`) : replayerCommand - replayerCommand = parseAndMergeArgs(replayerCommand, props.extraArgs); + command = props.extraArgs?.trim() ? command.concat(` ${props.extraArgs.trim()}`) : command this.createService({ serviceName: `traffic-replayer-${deployId}`, taskInstanceCount: 0, dockerDirectoryPath: join(__dirname, "../../../../../", "TrafficCapture/dockerSolution/build/docker/trafficReplayer"), - dockerImageCommand: ['/bin/sh', '-c', replayerCommand], + dockerImageCommand: ['/bin/sh', '-c', command], securityGroups: securityGroups, volumes: [sharedLogFileSystem.asVolume()], mountPoints: [sharedLogFileSystem.asMountPoint()], diff --git a/deployment/cdk/opensearch-service-migration/options.md b/deployment/cdk/opensearch-service-migration/options.md index 783d23ce3..fe9009bc4 100644 --- a/deployment/cdk/opensearch-service-migration/options.md +++ b/deployment/cdk/opensearch-service-migration/options.md @@ -142,8 +142,6 @@ A number of options are currently available but deprecated. While they function [^1]: Extra arguments can be added, overridden, or removed as follows: - - To add a new argument: Include the argument with the value, e.g., `"--new-arg value"` - - To override an existing argument: Include the argument with the new value, e.g., `"--override-arg new-value"` - - To remove an argument: Use the negated form, e.g., `"--no-existing-arg"` + - To add/override an argument: Include the argument with the value, e.g., `"--new-arg value"` + - Include quotes/escaping as appropriate for bash processing `"--new-arg \"my value\""` - Example: `"--new-arg value --existing-arg new-value --no-unwanted-arg"` diff --git a/deployment/cdk/opensearch-service-migration/package-lock.json b/deployment/cdk/opensearch-service-migration/package-lock.json index 818799260..ad90fd0ad 100644 --- a/deployment/cdk/opensearch-service-migration/package-lock.json +++ b/deployment/cdk/opensearch-service-migration/package-lock.json @@ -18,8 +18,7 @@ "node-forge": "^1.1.0", "semver": "^7.5.4", "source-map-support": "^0.5.21", - "yaml": "^2.4.3", - "yargs": "^17.7.2" + "yaml": "^2.4.3" }, "devDependencies": { "@aws-cdk/aws-msk-alpha": "^2.150.0-alpha.0", @@ -74,9 +73,9 @@ "license": "Apache-2.0" }, "node_modules/@aws-cdk/asset-node-proxy-agent-v6": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-node-proxy-agent-v6/-/asset-node-proxy-agent-v6-2.0.3.tgz", - "integrity": "sha512-twhuEG+JPOYCYPx/xy5uH2+VUsIEhPTzDY0F1KuB+ocjWWB/KEDiOVL19nHvbPCB6fhWnkykXEMJ4HHcKvjtvg==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-node-proxy-agent-v6/-/asset-node-proxy-agent-v6-2.1.0.tgz", + "integrity": "sha512-7bY3J8GCVxLupn/kNmpPc5VJz8grx+4RKfnnJiO1LG+uxkZfANZG3RMHhE+qQxxwkyQ9/MfPtTpf748UhR425A==", "dev": true, "license": "Apache-2.0" }, @@ -2707,6 +2706,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -2716,6 +2716,7 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, "license": "MIT", "dependencies": { "color-convert": "^2.0.1" @@ -2783,6 +2784,7 @@ "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.150.0.tgz", "integrity": "sha512-leo4J70QrJp+SYm/87VuoOVfALsW11F7JpkAGu5TLL/qd2k/CbovZ8k9/3Ov+jCVsvAgdn9DeHL01Sn6hSl6Zg==", "dev": true, + "license": "Apache-2.0", "bin": { "cdk": "bin/cdk" }, @@ -2811,6 +2813,7 @@ "mime-types" ], "dev": true, + "license": "Apache-2.0", "dependencies": { "@aws-cdk/asset-awscli-v1": "^2.2.202", "@aws-cdk/asset-kubectl-v20": "^2.1.2", @@ -3572,6 +3575,7 @@ "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, "license": "ISC", "dependencies": { "string-width": "^4.2.0", @@ -3604,6 +3608,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, "license": "MIT", "dependencies": { "color-name": "~1.1.4" @@ -3616,6 +3621,7 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, "license": "MIT" }, "node_modules/commander": { @@ -3821,6 +3827,7 @@ "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, "license": "MIT" }, "node_modules/error-ex": { @@ -3896,6 +3903,7 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz", "integrity": "sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==", + "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -4132,6 +4140,7 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, "license": "ISC", "engines": { "node": "6.* || 8.* || >= 10.*" @@ -4431,6 +4440,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -5725,6 +5735,7 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -5919,6 +5930,7 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, "license": "MIT", "dependencies": { "emoji-regex": "^8.0.0", @@ -5933,6 +5945,7 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, "license": "MIT", "dependencies": { "ansi-regex": "^5.0.1" @@ -6335,6 +6348,7 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^4.0.0", @@ -6395,6 +6409,7 @@ "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, "license": "ISC", "engines": { "node": ">=10" @@ -6423,6 +6438,7 @@ "version": "17.7.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dev": true, "license": "MIT", "dependencies": { "cliui": "^8.0.1", @@ -6441,6 +6457,7 @@ "version": "21.1.1", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, "license": "ISC", "engines": { "node": ">=12" diff --git a/deployment/cdk/opensearch-service-migration/package.json b/deployment/cdk/opensearch-service-migration/package.json index af12c2f59..67db9e0ef 100644 --- a/deployment/cdk/opensearch-service-migration/package.json +++ b/deployment/cdk/opensearch-service-migration/package.json @@ -66,8 +66,7 @@ "node-forge": "^1.1.0", "semver": "^7.5.4", "source-map-support": "^0.5.21", - "yaml": "^2.4.3", - "yargs": "^17.7.2" + "yaml": "^2.4.3" }, "peerDependencies": { "@aws-cdk/aws-servicecatalogappregistry-alpha": "^2.150.0-alpha.0", diff --git a/deployment/cdk/opensearch-service-migration/test/common-utilities.test.ts b/deployment/cdk/opensearch-service-migration/test/common-utilities.test.ts index 6c7a3659c..1ce27c069 100644 --- a/deployment/cdk/opensearch-service-migration/test/common-utilities.test.ts +++ b/deployment/cdk/opensearch-service-migration/test/common-utilities.test.ts @@ -1,119 +1,246 @@ import {CpuArchitecture} from "aws-cdk-lib/aws-ecs"; -import {parseAndMergeArgs, parseClusterDefinition, validateFargateCpuArch} from "../lib/common-utilities"; +import { + parseClusterDefinition, + validateFargateCpuArch, + parseArgsToDict, + appendArgIfNotInExtraArgs +} from "../lib/common-utilities"; import {describe, test, expect} from '@jest/globals'; -describe('validateFargateCpuArch', () => { - test('Test valid fargate cpu arch strings can be parsed', () => { - const cpuArch1 = "arm64" - const detectedArch1 = validateFargateCpuArch(cpuArch1) - expect(detectedArch1).toEqual(CpuArchitecture.ARM64) - - const cpuArch2 = "ARM64" - const detectedArch2 = validateFargateCpuArch(cpuArch2) - expect(detectedArch2).toEqual(CpuArchitecture.ARM64) - - const cpuArch3 = "x86_64" - const detectedArch3 = validateFargateCpuArch(cpuArch3) - expect(detectedArch3).toEqual(CpuArchitecture.X86_64) - - const cpuArch4 = "X86_64" - const detectedArch4 = validateFargateCpuArch(cpuArch4) - expect(detectedArch4).toEqual(CpuArchitecture.X86_64) - }) +describe('appendArgIfNotInExtraArgs', () => { - test('Test invalid fargate cpu arch strings throws error', () => { - const cpuArch = "arm32" - const getArchFunction = () => validateFargateCpuArch(cpuArch) - expect(getArchFunction).toThrow() - }) - - test('Test detected fargate cpu arch is valid', () => { - const detectedArch = process.arch - const detectedArchUpper = detectedArch.toUpperCase() - - const expectedCpuArch = detectedArchUpper === "X64" ? CpuArchitecture.X86_64 : CpuArchitecture.ARM64 - const cpuArch = validateFargateCpuArch() - expect(cpuArch).toEqual(expectedCpuArch) - }) - test('Test parseAndMergeArgs function', () => { - const baseCommand = 'node script.js --foo bar --baz --foo-bar bar'; - const extraArgs = '--qux quux --foo override'; - - const result = parseAndMergeArgs(baseCommand, extraArgs); + // Test when the arg is not present in extraArgsDict and has a value + test('appends arg and value when arg is not in extraArgsDict', () => { + const baseCommand = 'command'; + const extraArgsDict = { + "--arg1": ["value1"] + }; + const result = appendArgIfNotInExtraArgs(baseCommand, extraArgsDict, '--arg2', 'value2'); + expect(result).toBe('command --arg2 value2'); + }); - expect(result).toBe('node script.js --foo override --baz --foo-bar bar --qux quux'); + // Test when the arg is not present in extraArgsDict and has no value + test('appends arg without value when arg is not in extraArgsDict', () => { + const baseCommand = 'command'; + const extraArgsDict = { + "--arg1": ["value1"] + }; + const result = appendArgIfNotInExtraArgs(baseCommand, extraArgsDict, '--flag'); + expect(result).toBe('command --flag'); }); - test('Test parseAndMergeArgs function with only args', () => { - const baseCommand = '--foo bar --baz --foo-bar bar'; - const extraArgs = '--qux quux --foo override'; + // Test when the arg is already present in extraArgsDict (should not append) + test('does not append arg and value when arg is in extraArgsDict', () => { + const baseCommand = 'command'; + const extraArgsDict = { + "--arg1": ["value1"] + }; + const result = appendArgIfNotInExtraArgs(baseCommand, extraArgsDict, '--arg1', 'value1'); + expect(result).toBe('command'); // baseCommand should remain unchanged + }); - const result = parseAndMergeArgs(baseCommand, extraArgs); + // Test when extraArgsDict is empty (should append arg and value) + test('appends arg and value when extraArgsDict is empty', () => { + const baseCommand = 'command'; + const extraArgsDict = {}; + const result = appendArgIfNotInExtraArgs(baseCommand, extraArgsDict, '--arg1', 'value1'); + expect(result).toBe('command --arg1 value1'); + }); - expect(result).toBe('--foo override --baz --foo-bar bar --qux quux'); + // Test when extraArgsDict is empty and arg has no value (should append only arg) + test('appends only arg when extraArgsDict is empty and value is null', () => { + const baseCommand = 'command'; + const extraArgsDict = {}; + const result = appendArgIfNotInExtraArgs(baseCommand, extraArgsDict, '--flag'); + expect(result).toBe('command --flag'); + }); +}); + +describe('parseArgsToDict', () => { + + // Test valid input with multiple arguments + test('parses valid input with multiple arguments', () => { + const input = "--valid-arg some value --another-arg more values"; + const expectedOutput = { + "--valid-arg": ["some value"], + "--another-arg": ["more values"] + }; + expect(parseArgsToDict(input)).toEqual(expectedOutput); }); - test('parseAndMergeArgs handles boolean flags correctly', () => { - const baseCommand = 'node script.js --verbose --quiet false'; - const extraArgs = '--debug'; + // Test valid input with special characters in values + test('parses arguments with special characters in values', () => { + const input = "--valid-arg some!@--#$%^&*() value --another-arg value with spaces"; + const expectedOutput = { + "--valid-arg": ["some!@--#$%^&*() value"], + "--another-arg": ["value with spaces"] + }; + expect(parseArgsToDict(input)).toEqual(expectedOutput); + }); - const result = parseAndMergeArgs(baseCommand, extraArgs); + // Test when there are multiple spaces between argument and value + test('parses input with multiple spaces between argument and value', () => { + const input = "--valid-arg some value with spaces --another-arg more spaces"; + const expectedOutput = { + "--valid-arg": ["some value with spaces"], + "--another-arg": ["more spaces"] + }; + expect(parseArgsToDict(input)).toEqual(expectedOutput); + }); - expect(result).toBe('node script.js --verbose --quiet false --debug'); + // Test input with no value after an argument + test('handles argument with no value', () => { + const input = "--valid-arg --another-arg some value"; + const expectedOutput = { + "--valid-arg": [""], + "--another-arg": ["some value"] + }; + expect(parseArgsToDict(input)).toEqual(expectedOutput); }); - test('parseAndMergeArgs works without extra args', () => { - const baseCommand = 'node script.js --foo bar --baz'; + // Test input with argument at the start of the string + test('parses input with argument at the start of the string', () => { + const input = "--valid-arg start value --another-arg after value"; + const expectedOutput = { + "--valid-arg": ["start value"], + "--another-arg": ["after value"] + }; + expect(parseArgsToDict(input)).toEqual(expectedOutput); + }); - const result = parseAndMergeArgs(baseCommand); + // Test input with argument preceded by spaces + test('parses input where arguments are preceded by spaces', () => { + const input = " --valid-arg start value --another-arg after value"; + const expectedOutput = { + "--valid-arg": ["start value"], + "--another-arg": ["after value"] + }; + expect(parseArgsToDict(input)).toEqual(expectedOutput); + }); - expect(result).toBe('node script.js --foo bar --baz'); + // Test input with empty string + test('returns empty object for empty input', () => { + const input = ""; + const expectedOutput = {}; + expect(parseArgsToDict(input)).toEqual(expectedOutput); }); - test('parseAndMergeArgs propagates commands that start with --no in base command', () => { - const baseCommand = 'node script.js --no-verbose --debug'; - const extraArgs = '--foo bar'; + // Test input with argument -- + test('throws error for argument --', () => { + const input = "-- invalid arg some value"; + expect(() => parseArgsToDict(input)).toThrow("Invalid argument key: '--'. Argument keys must start with '--' and contain no spaces."); + }); - const result = parseAndMergeArgs(baseCommand, extraArgs); + // Test input with missing argument flag + test('throws error for missing argument flag', () => { + const input = "valid-arg some value"; + expect(() => parseArgsToDict(input)).toThrow("Invalid argument key: 'valid-arg'. Argument keys must start with '--' and contain no spaces."); + }); - expect(result).toBe('node script.js --no-verbose --debug --foo bar'); + // Test valid input with multiple special characters and whitespace + test('handles multiple spaces and special characters in value', () => { + const input = "--arg1 value with @#$%^&*! --arg2 multiple spaces"; + const expectedOutput = { + "--arg1": ["value with @#$%^&*!"], + "--arg2": ["multiple spaces"] + }; + expect(parseArgsToDict(input)).toEqual(expectedOutput); }); - test('parseAndMergeArgs negates commands that start with --no in base command', () => { - const baseCommand = 'node script.js --no-verbose'; - const extraArgs = '--no-no-verbose'; + // Test input with leading and trailing whitespace + test('trims leading and trailing whitespace from arguments and values', () => { + const input = " --valid-arg some value with spaces --another-arg more values "; + const expectedOutput = { + "--valid-arg": ["some value with spaces"], + "--another-arg": ["more values"] + }; + expect(parseArgsToDict(input)).toEqual(expectedOutput); + }); - const result = parseAndMergeArgs(baseCommand, extraArgs); + // Test input with only flags, no values + test('handles input with only flags and no values', () => { + const input = "--flag1 --flag2"; + const expectedOutput = { + "--flag1": [""], + "--flag2": [""] + }; + expect(parseArgsToDict(input)).toEqual(expectedOutput); + }); - expect(result).toBe('node script.js'); + // Test input with no space between flag and value + test('handles input with no space between flag and value', () => { + const input = "--flag1value --flag2"; + const expectedOutput = { + "--flag1value": [""], + "--flag2": [""] + }; + expect(parseArgsToDict(input)).toEqual(expectedOutput); }); - test('parseAndMergeArgs handles boolean negation in extra args removing boolean flags', () => { - const baseCommand = 'node script.js --verbose --debug'; - const extraArgs = '--no-verbose --foo bar'; + // Handles multiple occurrences of the same key + test('handles multiple occurrences of the same key', () => { + const input = "--key1 value1 --key1 value2 --key2 value3"; + const expectedOutput = { + "--key1": ["value1", "value2"], + "--key2": ["value3"] + }; + expect(parseArgsToDict(input)).toEqual(expectedOutput); + }); - const result = parseAndMergeArgs(baseCommand, extraArgs); + // Handles multiple occurrences of the same key with empty values + test('handles multiple occurrences of the same key with empty values', () => { + const input = "--key1 --key1 value2 --key2 value3"; + const expectedOutput = { + "--key1": ["", "value2"], + "--key2": ["value3"] + }; + expect(parseArgsToDict(input)).toEqual(expectedOutput); + }); - expect(result).toBe('node script.js --debug --foo bar'); + // Handles multiple occurrences of different keys + test('handles multiple occurrences of different keys', () => { + const input = "--key1 value1 --key2 value2 --key1 value3 --key2 value4"; + const expectedOutput = { + "--key1": ["value1", "value3"], + "--key2": ["value2", "value4"] + }; + expect(parseArgsToDict(input)).toEqual(expectedOutput); }); +}); - test('parseAndMergeArgs handles extraArgs boolean negations including only true flags in command', () => { - const baseCommand = 'node script.js --debug'; - const extraArgs = '--no-debug --verbose --quiet'; +describe('validateFargateCpuArch', () => { + test('Test valid fargate cpu arch strings can be parsed', () => { + const cpuArch1 = "arm64" + const detectedArch1 = validateFargateCpuArch(cpuArch1) + expect(detectedArch1).toEqual(CpuArchitecture.ARM64) - const result = parseAndMergeArgs(baseCommand, extraArgs); + const cpuArch2 = "ARM64" + const detectedArch2 = validateFargateCpuArch(cpuArch2) + expect(detectedArch2).toEqual(CpuArchitecture.ARM64) - expect(result).toBe('node script.js --verbose --quiet'); - }); + const cpuArch3 = "x86_64" + const detectedArch3 = validateFargateCpuArch(cpuArch3) + expect(detectedArch3).toEqual(CpuArchitecture.X86_64) - test('parseAndMergeArgs handles extraArgs boolean negations on non-boolean fields', () => { - const baseCommand = 'node script.js --foo bar'; - const extraArgs = '--no-foo'; + const cpuArch4 = "X86_64" + const detectedArch4 = validateFargateCpuArch(cpuArch4) + expect(detectedArch4).toEqual(CpuArchitecture.X86_64) + }) - const result = parseAndMergeArgs(baseCommand, extraArgs); + test('Test invalid fargate cpu arch strings throws error', () => { + const cpuArch = "arm32" + const getArchFunction = () => validateFargateCpuArch(cpuArch) + expect(getArchFunction).toThrow() + }) - expect(result).toBe('node script.js'); - }); + test('Test detected fargate cpu arch is valid', () => { + const detectedArch = process.arch + const detectedArchUpper = detectedArch.toUpperCase() + + const expectedCpuArch = detectedArchUpper === "X64" ? CpuArchitecture.X86_64 : CpuArchitecture.ARM64 + const cpuArch = validateFargateCpuArch() + expect(cpuArch).toEqual(expectedCpuArch) + }) test('parseClusterDefinition with basic auth parameters', () => { const clusterDefinition = { diff --git a/deployment/cdk/opensearch-service-migration/test/reindex-from-snapshot-stack.test.ts b/deployment/cdk/opensearch-service-migration/test/reindex-from-snapshot-stack.test.ts index 75e92f748..273892b67 100644 --- a/deployment/cdk/opensearch-service-migration/test/reindex-from-snapshot-stack.test.ts +++ b/deployment/cdk/opensearch-service-migration/test/reindex-from-snapshot-stack.test.ts @@ -115,7 +115,7 @@ describe('ReindexFromSnapshotStack Tests', () => { Value: { "Fn::Join": [ "", - [ "/rfs-app/runJavaWithClasspath.sh org.opensearch.migrations.RfsMigrateDocuments --s3-local-dir /tmp/s3_files --s3-repo-uri s3://migration-artifacts-test-account-unit-test-us-east-1/rfs-snapshot-repo --s3-region us-east-1 --snapshot-name rfs-snapshot --lucene-dir '/lucene' --target-host ", + [ "/rfs-app/runJavaWithClasspath.sh org.opensearch.migrations.RfsMigrateDocuments --s3-local-dir /tmp/s3_files --s3-repo-uri \"s3://migration-artifacts-test-account-unit-test-us-east-1/rfs-snapshot-repo\" --s3-region us-east-1 --snapshot-name rfs-snapshot --lucene-dir /lucene --target-host ", { "Ref": "SsmParameterValuemigrationunittestdefaultosClusterEndpointC96584B6F00A464EAD1953AFF4B05118Parameter", }, @@ -182,11 +182,11 @@ describe('ReindexFromSnapshotStack Tests', () => { Value: { "Fn::Join": [ "", - [ "/rfs-app/runJavaWithClasspath.sh org.opensearch.migrations.RfsMigrateDocuments --s3-local-dir /tmp/s3_files --s3-repo-uri s3://migration-artifacts-test-account-unit-test-us-east-1/rfs-snapshot-repo --s3-region us-east-1 --snapshot-name rfs-snapshot --lucene-dir '/lucene' --target-host ", + [ "/rfs-app/runJavaWithClasspath.sh org.opensearch.migrations.RfsMigrateDocuments --s3-local-dir /tmp/s3_files --s3-repo-uri \"s3://migration-artifacts-test-account-unit-test-us-east-1/rfs-snapshot-repo\" --s3-region us-east-1 --snapshot-name rfs-snapshot --lucene-dir /lucene --target-host ", { "Ref": "SsmParameterValuemigrationunittestdefaultosClusterEndpointC96584B6F00A464EAD1953AFF4B05118Parameter", }, - "--target-aws-service-signing-name aoss --target-aws-region eu-west-1", + " --target-aws-service-signing-name aoss --target-aws-region eu-west-1", ], ], } @@ -235,7 +235,7 @@ describe('ReindexFromSnapshotStack Tests', () => { expect(reindexStack.rfsSnapshotYaml.snapshot_name).toBe('rfs-snapshot'); }); - test('ReindexFromSnapshotStack correctly merges extraArgs', () => { + test('ReindexFromSnapshotStack correctly overrides with extraArgs', () => { const contextOptions = { vpcEnabled: true, reindexFromSnapshotServiceEnabled: true, @@ -244,7 +244,7 @@ describe('ReindexFromSnapshotStack Tests', () => { "endpoint": "https://test-cluster", "auth": {"type": "none"} }, - reindexFromSnapshotExtraArgs: '--custom-arg value --flag --snapshot-name custom-snapshot', + reindexFromSnapshotExtraArgs: '--custom-arg value --flag --snapshot-name \"custom-snapshot\"', migrationAssistanceEnabled: true, }; @@ -271,11 +271,11 @@ describe('ReindexFromSnapshotStack Tests', () => { Value: { "Fn::Join": [ "", - [ "/rfs-app/runJavaWithClasspath.sh org.opensearch.migrations.RfsMigrateDocuments --s3-local-dir /tmp/s3_files --s3-repo-uri s3://migration-artifacts-test-account-unit-test-us-east-1/rfs-snapshot-repo --s3-region us-east-1 --snapshot-name custom-snapshot --lucene-dir /lucene --target-host ", + [ "/rfs-app/runJavaWithClasspath.sh org.opensearch.migrations.RfsMigrateDocuments --s3-local-dir /tmp/s3_files --s3-repo-uri \"s3://migration-artifacts-test-account-unit-test-us-east-1/rfs-snapshot-repo\" --s3-region us-east-1 --lucene-dir /lucene --target-host ", { "Ref": "SsmParameterValuemigrationunittestdefaultosClusterEndpointC96584B6F00A464EAD1953AFF4B05118Parameter", }, - " --custom-arg value --flag" + " --custom-arg value --flag --snapshot-name \"custom-snapshot\"" ] ] }