Resolving Amazon Elastic Container Registry (ECR) Push Errors

  fail: docker push xyz.dkr.ecr.*.amazonaws.com/cdk-hnb659fds-container-assets-… exited with error code 1: failed commit on ref "manifest-sha256:" … : unexpected status from PUT request to https://xyz.dkr.ecr.*.amazonaws.com/v2/cdk-hnb659fds-container-assets.../manifests/: 400 Bad Request. Failed to publish asset

  Resource handler returned message: "Lambda function ... reached terminal FAILED state due to InvalidImage(ImageLayerFailure: UnsupportedImageLayerDetected - Layer Digest sha256:...) and failed to stabilize" (RequestToken: ..., HandlerErrorCode: NotStabilized)

export DOCKER_BUILDKIT=1
export BUILDKIT_PROGRESS=plain

alias 'docker build'='docker buildx build'

  docker system prune --all

is_command_present() {
    type "$1" >/dev/null 2>&1
}

install_jq() {
    printf "\n${YELLOW}PACKAGE CHECK:${NC}Ensuring that you are up to date on the following package: ${RED}jq${NC}\n"

    printf "\n${GREEN}Checking for jq installation.${NC}\n"

    # Check for jq installation
    if is_command_present "jq"; then
        echo "jq is already installed."
    else
        brew install jq
    fi
}

install_jq

#==============================
repositories=$(aws ecr describe-repositories --region $AWS_DEFAULT_REGION --query 'repositories[*].repositoryName' --output text)

for repo in $repositories; do
    echo "\nProcessing repository: $repo"

    # Get the list of image digests in the repository
    image_ids=$(aws ecr list-images --repository-name $repo --region $AWS_DEFAULT_REGION --query 'imageIds[*]' --output json)

    echo $image_ids | jq -c '.[]' | while read -r image; do
        digest=$(echo $image | jq -r '.imageDigest')

        # Get the image details
        image_details=$(aws ecr describe-images --repository-name $repo --image-ids imageDigest=$digest --region $AWS_DEFAULT_REGION)
        #echo "image_details for $tags:\n $image_details"

        # Get all tags for this image
        tags=$(echo $image_details | jq -r '.imageDetails[0].imageTags[]' 2>/dev/null | tr '\n' ',' | sed 's/,$//')

        image_size=$(echo $image_details | jq '.imageDetails[0].imageSizeInBytes')

        echo "\nImage digest: $digest\nRepository name: $repo \nImage tags: $tags"
        # Check if the image size is 0 bytes
        if [ "$image_size" = "0" ]; then
            echo "Image is empty, deleting image with digest $digest"
            aws ecr batch-delete-image --repository-name $repo --image-ids imageDigest=$digest --region $AWS_DEFAULT_REGION

            # Run the AWS ECR delete command
            output=$(aws ecr batch-delete-image --repository-name $repo --image-ids imageDigest=$digest --region $AWS_DEFAULT_REGION)

            # Check for failures or success
            echo "$output" | jq '
            if .failures | length > 0 then
                "Deletion Failure: \(.failures[0].failureReason)"
            else
                "Success: Image deleted"
            end'
        else
            echo "Image Size: $image_size bytes\nImage not empty [image not deleted]"
        fi
    done
done

// Old function ID
const prepareDataFn = new cdk.aws_lambda.DockerImageFunction(
  this,
  "PrepareDataLambdaV1Fn",
  {
    code: cdk.aws_lambda.DockerImageCode.fromImageAsset(
      path.join(__dirname, "path-to-docker-build-context")
    ),
  }
);

// New function ID
const prepareDataFn = new cdk.aws_lambda.DockerImageFunction(
  this,
  "PrepareDataLambdaV2Fn",
  {
    code: cdk.aws_lambda.DockerImageCode.fromImageAsset(
      path.join(__dirname, "path-to-docker-build-context")
    ),
  }
);

docker tag my_image_name: 241533140213.dkr.ecr.us-east-1.amazonaws.com/cdk-hnb659fds-container-assets-241533140213-us-east-1:

export BUILDX_NO_DEFAULT_ATTESTATIONS=1

DOCKER_BUILDKIT=1 docker build -t my_image_name --provenance=false --sbom=false .

import * as path from "path";
import * as cdk from "aws-cdk-lib";

const prepareDataFn = new cdk.aws_lambda.DockerImageFunction(
  this,
  "PrepareDataLambdaV1Fn",
  {
    code: cdk.aws_lambda.DockerImageCode.fromImageAsset(
      path.join("../../", "etl/prepare_data"),
      {
        platform: cdk.aws_ecr_assets.Platform.LINUX_ARM64,
        file: "Dockerfile",
        buildArgs: {
          provenance: "false",
          sbom: "false",
        },
      }
    ),
    architecture: cdk.aws_lambda.Architecture.ARM_64,
    description: `Prepare/Clean data - [${cdk.Stack.of(this).stackName}]`,
    timeout: cdk.Duration.minutes(15),
    memorySize: 1024,
  }
);

{
  "scripts": {
    "predeploy": "aws ecr get-login-password | docker login --username AWS --password-stdin $(aws sts get-caller-identity --query 'Account' --output text).dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com",
    "deploy": "DOCKER_BUILDKIT=1 sh -c 'cd my_cdk_stacks && npm run cdk deploy -- -c stack_name=\"${STACK_NAME:-}\" --all --require-approval never'",
    "destroy": "npm run cdk destroy -- -c stack_name=\"${STACK_NAME:-}\" --all"
  }
}

BUILDKIT_PROGRESS=plain CDK_VPC_ID=vpc-09f66ffff5d2773de STACK_NAME=MoyaTestStack npm run deploy

# Set variables
IMAGE_NAME="moya_test_image"
TAG="moya_test_image_tag"

REGION=$(aws configure get region --profile "${AWS_PROFILE}")
ACCOUNT_ID=$(aws sts get-caller-identity --query 'Account' --output text)

REPOSITORY_NAME="cdk-hnb659fds-container-assets-${ACCOUNT_ID}-${REGION}"

REGISTRY="${ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com"
FULL_IMAGE_NAME="${REGISTRY}/${REPOSITORY_NAME}:${TAG}"

echo "${FULL_IMAGE_NAME}"

# Build, tag, and push the Docker image

echo "Building Docker image ${IMAGE_NAME}..."

#ECR Issue Discussion - https://github.com/aws/aws-cdk/issues/31549
docker build -t "${IMAGE_NAME}" --provenance=false .

echo "Tagging image as ${FULL_IMAGE_NAME}..."
docker tag "${IMAGE_NAME}" "${FULL_IMAGE_NAME}"

# Login to ECR
echo "Logging into Amazon ECR..."
aws ecr get-login-password | docker login --username AWS --password-stdin $(aws sts get-caller-identity --query 'Account' --output text).dkr.ecr.${REGION}.amazonaws.com

echo "Checking if repository ${REPOSITORY_NAME} exists in ECR..."

# Use aws CLI to check if the repository exists
if aws ecr describe-repositories \
    --repository-names "${REPOSITORY_NAME}" \
    --region "${REGION}" \
    --profile "${AWS_PROFILE}" > /dev/null 2>&1; then
    echo "Repository ${REPOSITORY_NAME} already exists."
else
    echo "Repository ${REPOSITORY_NAME} does not exist in ECR."
fi

echo "Pushing image to ECR..."
docker push "${FULL_IMAGE_NAME}"

{
  "name": "project-root",
  "version": "0.0.1",
  "scripts": {
    "deploy": "DOCKER_BUILDKIT=1 sh -c 'cd my_cdk_stacks && npm run cdk deploy -- -c stack_name=\"${STACK_NAME:-}\" --all --require-approval never'",
    "predeploy": "aws ecr get-login-password | docker login --username AWS --password-stdin $(aws sts get-caller-identity --query 'Account' --output text).dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com",
    "destroy": "cd my_cdk_stacks && npm run cdk destroy -- -c stack_name=\"${STACK_NAME:-}\" --all",
    "predeploy.moya.test": "aws ecr get-login-password | docker login --username AWS --password-stdin $(aws sts get-caller-identity --query 'Account' --output text).dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com",
    "deploy.moya.test": "DOCKER_BUILDKIT=1 sh -c 'echo \"moyas--$DOCKER_BUILDKIT\" && cd my_cdk_stacks && npm install && cdk deploy --all --require-approval never --app \"npx ts-node src/moya-test-stack.ts\" --context \"stack_name=${STACK_NAME}\"'",
    "destroy.moya.test": "cd my_cdk_stacks && npx cdk destroy --app \"npx ts-node src/moya-test-stack.ts\" -c stack_name=\"${STACK_NAME:-}\" --all"
  }
}

CDK_VPC_ID=vpc-09f66ffff5d2793de STACK_NAME=MoyaTestStack npm run deploy.moya.test

FROM public.ecr.aws/lambda/python:3.12

# Set the working directory inside the container to Lambda's task root
WORKDIR ${LAMBDA_TASK_ROOT}

# Copy requirements.txt first to leverage Docker's layer caching
COPY requirements.txt .

# Install the specified packages into the Lambda task root
RUN pip3 install --target "${LAMBDA_TASK_ROOT}" -r requirements.txt --no-cache-dir

# Copy the rest of the application code to the Lambda task root
COPY . .

# Set the CMD to the Lambda handler
CMD [ "index.lambda_handler" ]

import * as path from "path";
import * as cdk from "aws-cdk-lib";
import { Construct } from "constructs";
import { AwsSolutionsChecks } from "cdk-nag";
import { NagSuppressions } from "cdk-nag/lib/nag-suppressions";

import * as dotenv from "dotenv";
// Load environment variables from a .env file
dotenv.config();

export interface MoyaTestStackProps extends cdk.StackProps {
  vpc?: cdk.aws_ec2.IVpc;
}

export class MoyaTestStack extends cdk.Stack {
  props: MoyaTestStackProps;

  constructor(scope: Construct, id: string, props: MoyaTestStackProps) {
    super(scope, id, props);
    this.props = props;

    const vpc = props?.vpc ?? this.retrieveVpc();

    const backendPythonProjectRoot = path.join(
      __dirname,
      "../../backend/python"
    );

    const prepareDataFn = new cdk.aws_lambda.DockerImageFunction(
      this,
      "PrepareMoyaDataLambdV3FnV1",
      {
        code: cdk.aws_lambda.DockerImageCode.fromImageAsset(
          path.join(backendPythonProjectRoot, "etl/prepare_data"),
          {
            platform: cdk.aws_ecr_assets.Platform.LINUX_ARM64,
            file: "Dockerfile",
            cmd: ["index.lambda_handler"],
            buildArgs: {
              // Issue Discussion: https://github.com/aws/aws-cdk/issues/31548
              // Fix for ECR push error - [Failed to push image: failed commit on ref, unexpected status from PUT request, 400 Bad Request]
              provenance: "false",
              sbom: "false",
            },
          }
        ),

        architecture: cdk.aws_lambda.Architecture.ARM_64,
        description: `Prepare/Clean data - [${cdk.Stack.of(this).stackName}]`,
        timeout: cdk.Duration.minutes(15),
        memorySize: 1024,
        vpc: vpc,
        vpcSubnets: vpc.selectSubnets({
          subnetType: cdk.aws_ec2.SubnetType.PRIVATE_WITH_EGRESS,
        }),
      }
    );

    this.suppressNags();
  }
  /**
   * Retrieves the VPC (Virtual Private Cloud) based on the provided VPC ID.
   * Throws an error if the VPC ID is not provided via the environment variable CDK_VPC_ID.
   *
   * @returns {cdk.aws_ec2.IVpc} The VPC object retrieved from AWS CDK.
   * @throws {Error} Throws an error if CDK_VPC_ID environment variable is not set.
   */
  retrieveVpc() {
    const stack = cdk.Stack.of(this);
    const env = stack.node.tryGetContext("env");
    const vpcId = process.env.CDK_VPC_ID || env?.vpcId;

    if (!vpcId) {
      throw new Error("CDK_VPC_ID environment variable not set");
    }

    const vpc = cdk.aws_ec2.Vpc.fromLookup(this, "Vpc", {
      vpcId: vpcId,
    });

    return vpc;
  }
  private suppressNags(): void {
    const stack = cdk.Stack.of(this);

    // Suppressions for AWS CDK Nag tool
    NagSuppressions.addStackSuppressions(
      stack,
      [
        {
          id: "AwsSolutions-IAM4",
          reason: "AWS managed policies are permitted for demo purposes",
          appliesTo: [
            // Specify the AWS managed policies allowed in this CDK stack.
            // These policies must be specific and non-overly permissive.
            // For example, avoid policies like AmazonS3FullAccess, AmazonSageMakerFullAccess.
            "Policy::arn::iam::aws:policy/service-role/AWSLambdaBasicExecutionRole",
            "Policy::arn::iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole",
          ],
        },
      ],
      true
    );
  }
}

//-----------------------------
// CDK deploy
//-----------------------------

const app = new cdk.App();
const stackName = app.node.tryGetContext("stack_name") || "MoyaTestStack";
const account =
  app.node.tryGetContext("account") ||
  process.env.CDK_DEPLOY_ACCOUNT ||
  process.env.CDK_DEFAULT_ACCOUNT;
const region =
  app.node.tryGetContext("region") ||
  process.env.CDK_DEPLOY_REGION ||
  process.env.CDK_DEFAULT_REGION;

const moyaTestStack = new MoyaTestStack(app, stackName, {
  env: {
    account: account,
    region: region,
  },
});

cdk.Aspects.of(app).add(
  new AwsSolutionsChecks({ logIgnores: true, verbose: true })
);
app.synth();