DeleteConflict Error Code on sls remove

Hi,

I am experiencing an issue where I can deploy the basic aws-nodejs template with no issues, but when executing an sls remove, I get the following error:

An error occurred: IamRoleLambdaExecution - Cannot delete entity, must delete policies first. (Service: AmazonIdentityManagement; Status Code: 409; Error Code: DeleteConflict; Request ID: ********).

I have verified via aws cli that the user can successfully delete the role and its inline policy if done in the proper order:

  1. aws iam delete-role-policy --role-name aws-nodejs-dev-us-east-1-lambdaRole --policy-name dev-aws-nodejs-lambda
  2. aws iam delete-role --role-name aws-nodejs-dev-us-east-1-lambdaRole

Any idea why this might be happening?

Policy

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "CloudFormation",
            "Effect": "Allow",
            "Action": [
                "cloudformation:DescribeChangeSet",
                "cloudformation:CreateStack",
                "cloudformation:DeleteStack",
                "cloudformation:UpdateStack",
                "cloudformation:DescribeStacks",
                "cloudformation:DescribeStackEvents",
                "cloudformation:DescribeStackResource"
            ],
            "Resource": "arn:aws:cloudformation:*:*"
        },
        {
            "Sid": "CloudFormationAllResources",
            "Effect": "Allow",
            "Action": [
                "cloudformation:ValidateTemplate",
                "cloudformation:DescribeAccountLimits",
                "cloudformation:ListStacks",
                "cloudformation:ListImports",
                "cloudformation:ListExports",
                "cloudformation:EstimateTemplateCost",
                "cloudformation:GetTemplateSummary",
                "cloudformation:CreateUploadBucket"
            ],
            "Resource": "*"
        },
        {
            "Sid": "S3",
            "Effect": "Allow",
            "Action": [
                "s3:CreateBucket",
                "s3:PutObject",
                "s3:GetObject",
                "s3:DeleteObject",
                "s3:DeleteBucket",
                "s3:ListBucketVersions",
                "s3:ListBucket"
            ],
            "Resource": "arn:aws:s3::*"
        },
        {
            "Sid": "Logs",
            "Effect": "Allow",
            "Action": [
                "logs:DescribeLogGroups",
                "logs:DescribeLogStreams",
                "logs:CreateLogGroup",
                "logs:DeleteLogGroup"
            ],
            "Resource": "arn:aws:logs:*:*:*"
        },
        {
            "Sid": "IAM",
            "Effect": "Allow",
            "Action": [
                "iam:GetRole",
                "iam:CreateRole",
                "iam:DeleteRole",
                "iam:PutRolePolicy",
                "iam:DetachRolePolicy",
                "iam:DeleteRolePolicy",
                "iam:PassRole"
            ],
            "Resource": "arn:aws:iam::*:role/*-lambdaRole"
        },
        {
            "Sid": "Lambda",
            "Effect": "Allow",
            "Action": [
                "lambda:*"
            ],
            "Resource": "*"
        }
    ]
}

Resulting CloudFormation Template

{
   "AWSTemplateFormatVersion":"2010-09-09",
   "Description":"The AWS CloudFormation template for this Serverless application",
   "Resources":{
      "ServerlessDeploymentBucket":{
         "Type":"AWS::S3::Bucket"
      },
      "HelloLogGroup":{
         "Type":"AWS::Logs::LogGroup",
         "Properties":{
            "LogGroupName":"/aws/lambda/aws-nodejs-dev-hello"
         }
      },
      "IamRoleLambdaExecution":{
         "Type":"AWS::IAM::Role",
         "Properties":{
            "AssumeRolePolicyDocument":{
               "Version":"2012-10-17",
               "Statement":[
                  {
                     "Effect":"Allow",
                     "Principal":{
                        "Service":[
                           "lambda.amazonaws.com"
                        ]
                     },
                     "Action":[
                        "sts:AssumeRole"
                     ]
                  }
               ]
            },
            "Policies":[
               {
                  "PolicyName":{
                     "Fn::Join":[
                        "-",
                        [
                           "dev",
                           "aws-nodejs",
                           "lambda"
                        ]
                     ]
                  },
                  "PolicyDocument":{
                     "Version":"2012-10-17",
                     "Statement":[
                        {
                           "Effect":"Allow",
                           "Action":[
                              "logs:CreateLogStream"
                           ],
                           "Resource":[
                              {
                                 "Fn::Sub":"arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/aws-nodejs-dev-hello:*"
                              }
                           ]
                        },
                        {
                           "Effect":"Allow",
                           "Action":[
                              "logs:PutLogEvents"
                           ],
                           "Resource":[
                              {
                                 "Fn::Sub":"arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/aws-nodejs-dev-hello:*:*"
                              }
                           ]
                        }
                     ]
                  }
               }
            ],
            "Path":"/",
            "RoleName":{
               "Fn::Join":[
                  "-",
                  [
                     "aws-nodejs",
                     "dev",
                     "us-east-1",
                     "lambdaRole"
                  ]
               ]
            }
         }
      },
      "HelloLambdaFunction":{
         "Type":"AWS::Lambda::Function",
         "Properties":{
            "Code":{
               "S3Bucket":{
                  "Ref":"ServerlessDeploymentBucket"
               },
               "S3Key":"serverless/aws-nodejs/dev/1529426978047-2018-06-19T16:49:38.047Z/aws-nodejs.zip"
            },
            "FunctionName":"aws-nodejs-dev-hello",
            "Handler":"handler.hello",
            "MemorySize":1024,
            "Role":{
               "Fn::GetAtt":[
                  "IamRoleLambdaExecution",
                  "Arn"
               ]
            },
            "Runtime":"nodejs6.10",
            "Timeout":6
         },
         "DependsOn":[
            "HelloLogGroup",
            "IamRoleLambdaExecution"
         ]
      },
      "HelloLambdaVersionPAdpuBLiPspWjaXvfnsI8PkMWAClV0kFQY9j9iA":{
         "Type":"AWS::Lambda::Version",
         "DeletionPolicy":"Retain",
         "Properties":{
            "FunctionName":{
               "Ref":"HelloLambdaFunction"
            },
            "CodeSha256":"PSzzisjnTvvYknuXw+QOlAvdkQZ67qXYSvgoAi9T8W0="
         }
      }
   },
   "Outputs":{
      "ServerlessDeploymentBucketName":{
         "Value":{
            "Ref":"ServerlessDeploymentBucket"
         }
      },
      "HelloLambdaFunctionQualifiedArn":{
         "Description":"Current Lambda function version",
         "Value":{
            "Ref":"HelloLambdaVersionPAdpuBLiPspWjaXvfnsI8PkMWAClV0kFQY9j9iA"
         }
      }
   }
}

Environment:
sls v1.27.3
Ubuntu 16.0.4.3 LTS
Node v8.11.1

Did you make any changes to the role after you deployed? Serverless uses a CloudFormation stack to build the environment and any changes through the CLI or UI can cause the two to get out of sync.

1 Like

I didn’t make any changes to the role, I’m only using the deploy and remove commands of the cli.

When I look at the stack events from cloud formation, I see the role creation, but I don’t see any event related to policy…does this mean the cli is creating the inline policy outside of cloudformation?

Stack events for aws-nodejs-dev

2018-06-19 Status Type Logical ID Status Reason
12:17:27 UTC-0400 UPDATE_COMPLETE AWS::CloudFormation::Stack aws-nodejs-dev
12:17:26 UTC-0400 UPDATE_COMPLETE_CLEANUP_IN_PROGRESS AWS::CloudFormation::Stack aws-nodejs-dev
12:17:24 UTC-0400 CREATE_COMPLETE AWS::Lambda::Version HelloLambdaVersionPAdpuBLiPspWjaXvfnsI8PkMWAClV0kFQY9j9iA
12:17:24 UTC-0400 CREATE_IN_PROGRESS AWS::Lambda::Version HelloLambdaVersionPAdpuBLiPspWjaXvfnsI8PkMWAClV0kFQY9j9iA Resource creation Initiated
12:17:23 UTC-0400 CREATE_IN_PROGRESS AWS::Lambda::Version HelloLambdaVersionPAdpuBLiPspWjaXvfnsI8PkMWAClV0kFQY9j9iA
12:17:21 UTC-0400 CREATE_COMPLETE AWS::Lambda::Function HelloLambdaFunction
12:17:20 UTC-0400 CREATE_IN_PROGRESS AWS::Lambda::Function HelloLambdaFunction Resource creation Initiated
12:17:19 UTC-0400 CREATE_IN_PROGRESS AWS::Lambda::Function HelloLambdaFunction
12:17:17 UTC-0400 CREATE_COMPLETE AWS::IAM::Role IamRoleLambdaExecution
12:17:01 UTC-0400 CREATE_COMPLETE AWS::Logs::LogGroup HelloLogGroup
12:17:00 UTC-0400 CREATE_IN_PROGRESS AWS::Logs::LogGroup HelloLogGroup Resource creation Initiated
12:17:00 UTC-0400 CREATE_IN_PROGRESS AWS::IAM::Role IamRoleLambdaExecution Resource creation Initiated
12:16:59 UTC-0400 CREATE_IN_PROGRESS AWS::IAM::Role IamRoleLambdaExecution
12:16:59 UTC-0400 CREATE_IN_PROGRESS AWS::Logs::LogGroup HelloLogGroup
12:16:56 UTC-0400 UPDATE_IN_PROGRESS AWS::CloudFormation::Stack aws-nodejs-dev User Initiated
12:16:52 UTC-0400 CREATE_COMPLETE AWS::CloudFormation::Stack aws-nodejs-dev
12:16:51 UTC-0400 CREATE_COMPLETE AWS::S3::Bucket ServerlessDeploymentBucket
12:16:30 UTC-0400 CREATE_IN_PROGRESS AWS::S3::Bucket ServerlessDeploymentBucket Resource creation Initiated
12:16:29 UTC-0400 CREATE_IN_PROGRESS AWS::S3::Bucket ServerlessDeploymentBucket
12:16:26 UTC-0400 CREATE_IN_PROGRESS AWS::CloudFormation::Stack aws-nodejs-dev User Initiated

The IAM Role created has an inline policy. That’s part of the stack. Have you tried deleting the stack from the CloudFormation console?

1 Like

Deleting the stack manually from the console results in the same error. I tested out a deploy and remove with a new service name (aws-nodejs1-dev vs aws-nodejs-dev) and it worked fine, so it must be an issue on the AWS side.

In any case, thank you for your help, much appreciated!

1 Like