AWSTemplateFormatVersion: '2010-09-09'
Description: 'Secure Data Lake with S3, Glue, Athena, and Lake Formation'

Parameters:
  Environment:
    Type: String
    Default: dev
    AllowedValues: [dev, prod]
  DataLakeAdminArn:
    Type: String
    Description: IAM User or Role ARN to be granted Lake Formation Data Lake Admin

Resources:
  # KMS Key
  DataLakeKey:
    Type: AWS::KMS::Key
    Properties:
      Description: KMS key for data lake encryption
      EnableKeyRotation: true
      KeyPolicy:
        Version: '2012-10-17'
        Statement:
          - Sid: Enable IAM User Permissions
            Effect: Allow
            Principal:
              AWS: !Sub 'arn:aws:iam::${AWS::AccountId}:root'
            Action: 'kms:*'
            Resource: '*'

  DataLakeKeyAlias:
    Type: AWS::KMS::Alias
    Properties:
      AliasName: !Sub 'alias/data-lake-${Environment}'
      TargetKeyId: !Ref DataLakeKey

  # S3 Buckets
  RawDataBucket:
    Type: AWS::S3::Bucket
    Properties:
      BucketEncryption:
        ServerSideEncryptionConfiguration:
          - ServerSideEncryptionByDefault:
              SSEAlgorithm: aws:kms
              KMSMasterKeyID: !Ref DataLakeKey
            BucketKeyEnabled: true
      PublicAccessBlockConfiguration:
        BlockPublicAcls: true
        BlockPublicPolicy: true
        IgnorePublicAcls: true
        RestrictPublicBuckets: true
      VersioningConfiguration:
        Status: Enabled
      LifecycleConfiguration:
        Rules:
          - Id: TransitionToIA
            Status: Enabled
            Transitions:
              - TransitionInDays: 90
                StorageClass: STANDARD_IA
              - TransitionInDays: 365
                StorageClass: GLACIER
      Tags:
        - Key: Environment
          Value: !Ref Environment
        - Key: DataClassification
          Value: raw

  ProcessedDataBucket:
    Type: AWS::S3::Bucket
    Properties:
      BucketEncryption:
        ServerSideEncryptionConfiguration:
          - ServerSideEncryptionByDefault:
              SSEAlgorithm: aws:kms
              KMSMasterKeyID: !Ref DataLakeKey
            BucketKeyEnabled: true
      PublicAccessBlockConfiguration:
        BlockPublicAcls: true
        BlockPublicPolicy: true
        IgnorePublicAcls: true
        RestrictPublicBuckets: true
      VersioningConfiguration:
        Status: Enabled
      Tags:
        - Key: Environment
          Value: !Ref Environment
        - Key: DataClassification
          Value: processed

  AthenaResultsBucket:
    Type: AWS::S3::Bucket
    Properties:
      BucketEncryption:
        ServerSideEncryptionConfiguration:
          - ServerSideEncryptionByDefault:
              SSEAlgorithm: aws:kms
              KMSMasterKeyID: !Ref DataLakeKey
            BucketKeyEnabled: true
      PublicAccessBlockConfiguration:
        BlockPublicAcls: true
        BlockPublicPolicy: true
        IgnorePublicAcls: true
        RestrictPublicBuckets: true
      LifecycleConfiguration:
        Rules:
          - Id: ExpireQueryResults
            Status: Enabled
            ExpirationInDays: 30
      Tags:
        - Key: Environment
          Value: !Ref Environment

  # Lake Formation Settings
  DataLakeSettings:
    Type: AWS::LakeFormation::DataLakeSettings
    Properties:
      Admins:
        - DataLakePrincipalIdentifier: !Ref DataLakeAdminArn

  # Glue Database
  GlueDatabase:
    Type: AWS::Glue::Database
    Properties:
      CatalogId: !Ref AWS::AccountId
      DatabaseInput:
        Name: !Sub 'data_lake_${Environment}'
        Description: Data lake database for analytics

  # IAM Role for Glue
  GlueServiceRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: glue.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole
      Policies:
        - PolicyName: GlueDataLakePolicy
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - s3:GetObject
                  - s3:PutObject
                  - s3:DeleteObject
                  - s3:ListBucket
                Resource:
                  - !GetAtt RawDataBucket.Arn
                  - !Sub '${RawDataBucket.Arn}/*'
                  - !GetAtt ProcessedDataBucket.Arn
                  - !Sub '${ProcessedDataBucket.Arn}/*'
              - Effect: Allow
                Action:
                  - kms:Decrypt
                  - kms:GenerateDataKey
                Resource: !GetAtt DataLakeKey.Arn
              - Effect: Allow
                Action:
                  - lakeformation:GetDataAccess
                Resource: '*'

  # Glue Crawler for raw data
  RawDataCrawler:
    Type: AWS::Glue::Crawler
    Properties:
      Name: !Sub 'raw-data-crawler-${Environment}'
      Role: !GetAtt GlueServiceRole.Arn
      DatabaseName: !Ref GlueDatabase
      Targets:
        S3Targets:
          - Path: !Sub 's3://${RawDataBucket}/data/'
      SchemaChangePolicy:
        UpdateBehavior: UPDATE_IN_DATABASE
        DeleteBehavior: LOG
      Schedule:
        ScheduleExpression: 'cron(0 2 * * ? *)'

  # Glue ETL Job
  GlueETLJob:
    Type: AWS::Glue::Job
    Properties:
      Name: !Sub 'etl-job-${Environment}'
      Role: !GetAtt GlueServiceRole.Arn
      GlueVersion: '4.0'
      WorkerType: G.1X
      NumberOfWorkers: 2
      Timeout: 60
      DefaultArguments:
        '--job-language': python
        '--enable-metrics': 'true'
        '--enable-continuous-cloudwatch-log': 'true'
        '--enable-spark-ui': 'true'
        '--TempDir': !Sub 's3://${ProcessedDataBucket}/tmp/'
        '--SOURCE_BUCKET': !Ref RawDataBucket
        '--TARGET_BUCKET': !Ref ProcessedDataBucket
      Command:
        Name: glueetl
        ScriptLocation: !Sub 's3://${ProcessedDataBucket}/scripts/etl.py'
        PythonVersion: '3'

  # Athena Workgroup
  AthenaWorkgroup:
    Type: AWS::Athena::WorkGroup
    Properties:
      Name: !Sub 'data-lake-workgroup-${Environment}'
      Description: Workgroup for data lake queries
      WorkGroupConfiguration:
        EnforceWorkGroupConfiguration: true
        ResultConfiguration:
          OutputLocation: !Sub 's3://${AthenaResultsBucket}/results/'
          EncryptionConfiguration:
            EncryptionOption: SSE_KMS
            KmsKey: !Ref DataLakeKey
        BytesScannedCutoffPerQuery: 10737418240  # 10 GB
        PublishCloudWatchMetricsEnabled: true

  # IAM Role for Athena users
  AthenaAnalystRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              AWS: !Sub 'arn:aws:iam::${AWS::AccountId}:root'
            Action: sts:AssumeRole
      Policies:
        - PolicyName: AthenaQueryPolicy
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - athena:StartQueryExecution
                  - athena:GetQueryExecution
                  - athena:GetQueryResults
                  - athena:StopQueryExecution
                  - athena:ListQueryExecutions
                Resource: !Sub 'arn:aws:athena:${AWS::Region}:${AWS::AccountId}:workgroup/${AthenaWorkgroup}'
              - Effect: Allow
                Action:
                  - glue:GetDatabase
                  - glue:GetTable
                  - glue:GetTables
                  - glue:GetPartition
                  - glue:GetPartitions
                Resource:
                  - !Sub 'arn:aws:glue:${AWS::Region}:${AWS::AccountId}:catalog'
                  - !Sub 'arn:aws:glue:${AWS::Region}:${AWS::AccountId}:database/${GlueDatabase}'
                  - !Sub 'arn:aws:glue:${AWS::Region}:${AWS::AccountId}:table/${GlueDatabase}/*'
              - Effect: Allow
                Action:
                  - s3:GetObject
                  - s3:ListBucket
                Resource:
                  - !GetAtt ProcessedDataBucket.Arn
                  - !Sub '${ProcessedDataBucket.Arn}/*'
              - Effect: Allow
                Action:
                  - s3:GetObject
                  - s3:PutObject
                  - s3:ListBucket
                Resource:
                  - !GetAtt AthenaResultsBucket.Arn
                  - !Sub '${AthenaResultsBucket.Arn}/*'
              - Effect: Allow
                Action:
                  - kms:Decrypt
                  - kms:GenerateDataKey
                Resource: !GetAtt DataLakeKey.Arn
              - Effect: Allow
                Action:
                  - lakeformation:GetDataAccess
                Resource: '*'

  # Lake Formation permissions for Glue role
  GlueLakeFormationPermission:
    Type: AWS::LakeFormation::Permissions
    DependsOn: DataLakeSettings
    Properties:
      DataLakePrincipal:
        DataLakePrincipalIdentifier: !GetAtt GlueServiceRole.Arn
      Resource:
        DatabaseResource:
          Name: !Ref GlueDatabase
      Permissions:
        - ALL

  # CloudWatch Alarms
  GlueJobFailureAlarm:
    Type: AWS::CloudWatch::Alarm
    Properties:
      AlarmDescription: Alert when Glue ETL job fails
      MetricName: glue.driver.aggregate.numFailedTasks
      Namespace: Glue
      Statistic: Sum
      Period: 300
      EvaluationPeriods: 1
      Threshold: 1
      ComparisonOperator: GreaterThanOrEqualToThreshold

Outputs:
  RawDataBucketName:
    Description: S3 bucket for raw data ingestion
    Value: !Ref RawDataBucket
    Export:
      Name: !Sub '${AWS::StackName}-RawDataBucket'

  ProcessedDataBucketName:
    Description: S3 bucket for processed/curated data
    Value: !Ref ProcessedDataBucket
    Export:
      Name: !Sub '${AWS::StackName}-ProcessedDataBucket'

  GlueDatabaseName:
    Description: Glue Data Catalog database name
    Value: !Ref GlueDatabase
    Export:
      Name: !Sub '${AWS::StackName}-GlueDatabase'

  AthenaWorkgroupName:
    Description: Athena workgroup for querying data lake
    Value: !Ref AthenaWorkgroup
    Export:
      Name: !Sub '${AWS::StackName}-AthenaWorkgroup'

  AthenaAnalystRoleArn:
    Description: IAM Role ARN for Athena analysts
    Value: !GetAtt AthenaAnalystRole.Arn
    Export:
      Name: !Sub '${AWS::StackName}-AthenaAnalystRole'