Handism's Tech Blog
一覧に戻る
Analytics

セキュアなデータレイク構成

S3、AWS Glue、Amazon Athena、Lake Formation を組み合わせ、きめ細かなアクセス制御と KMS 暗号化を施したバッチ分析・クエリ基盤を構築します。Glue Crawler による自動スキーマ検出 and Athena ワークグループによるクエリコスト管理も含みます。

構成要素 (AWS Services):

S3GlueAthenaLake FormationKMSIAM

構築される主要リソース (11種):

S3 Bucket3
IAM Role2
KMS Key1
KMS Alias1
LakeFormation DataLakeSettings1
Glue Database1
Glue Crawler1
Glue Job1
Athena WorkGroup1
LakeFormation Permissions1
CloudWatch Alarm1

アーキテクチャ図 (Architecture Diagram)

クリックで拡大表示
セキュアなデータレイク構成 アーキテクチャ図

AWS CLI でのデプロイ例

AWS CLIを使用してCloudFormationスタックをデプロイする場合は、以下のコマンドを実行します。

aws cloudformation deploy \
  --template-file secure-data-lake.yaml \
  --stack-name secure-data-lake-stack \
  --capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM
secure-data-lake.yaml
DL
AWSTemplateFormatVersion: '2010-09-09'
Description: 'Secure Data Lake with S3, Glue, Athena, and Lake Formation'

Parameters:
  Environment:
    Type: String
    Default: dev
    AllowedValues: [dev, prod]
  DataLakeAdminArn:
    Type: String
    Description: IAM User or Role ARN to be granted Lake Formation Data Lake Admin

Resources:
  # KMS Key
  DataLakeKey:
    Type: AWS::KMS::Key
    Properties:
      Description: KMS key for data lake encryption
      EnableKeyRotation: true
      KeyPolicy:
        Version: '2012-10-17'
        Statement:
          - Sid: Enable IAM User Permissions
            Effect: Allow
            Principal:
              AWS: !Sub 'arn:aws:iam::${AWS::AccountId}:root'
            Action: 'kms:*'
            Resource: '*'

  DataLakeKeyAlias:
    Type: AWS::KMS::Alias
    Properties:
      AliasName: !Sub 'alias/data-lake-${Environment}'
      TargetKeyId: !Ref DataLakeKey

  # S3 Buckets
  RawDataBucket:
    Type: AWS::S3::Bucket
    Properties:
      BucketEncryption:
        ServerSideEncryptionConfiguration:
          - ServerSideEncryptionByDefault:
              SSEAlgorithm: aws:kms
              KMSMasterKeyID: !Ref DataLakeKey
            BucketKeyEnabled: true
      PublicAccessBlockConfiguration:
        BlockPublicAcls: true
        BlockPublicPolicy: true
        IgnorePublicAcls: true
        RestrictPublicBuckets: true
      VersioningConfiguration:
        Status: Enabled
      LifecycleConfiguration:
        Rules:
          - Id: TransitionToIA
            Status: Enabled
            Transitions:
              - TransitionInDays: 90
                StorageClass: STANDARD_IA
              - TransitionInDays: 365
                StorageClass: GLACIER
      Tags:
        - Key: Environment
          Value: !Ref Environment
        - Key: DataClassification
          Value: raw

  ProcessedDataBucket:
    Type: AWS::S3::Bucket
    Properties:
      BucketEncryption:
        ServerSideEncryptionConfiguration:
          - ServerSideEncryptionByDefault:
              SSEAlgorithm: aws:kms
              KMSMasterKeyID: !Ref DataLakeKey
            BucketKeyEnabled: true
      PublicAccessBlockConfiguration:
        BlockPublicAcls: true
        BlockPublicPolicy: true
        IgnorePublicAcls: true
        RestrictPublicBuckets: true
      VersioningConfiguration:
        Status: Enabled
      Tags:
        - Key: Environment
          Value: !Ref Environment
        - Key: DataClassification
          Value: processed

  AthenaResultsBucket:
    Type: AWS::S3::Bucket
    Properties:
      BucketEncryption:
        ServerSideEncryptionConfiguration:
          - ServerSideEncryptionByDefault:
              SSEAlgorithm: aws:kms
              KMSMasterKeyID: !Ref DataLakeKey
            BucketKeyEnabled: true
      PublicAccessBlockConfiguration:
        BlockPublicAcls: true
        BlockPublicPolicy: true
        IgnorePublicAcls: true
        RestrictPublicBuckets: true
      LifecycleConfiguration:
        Rules:
          - Id: ExpireQueryResults
            Status: Enabled
            ExpirationInDays: 30
      Tags:
        - Key: Environment
          Value: !Ref Environment

  # Lake Formation Settings
  DataLakeSettings:
    Type: AWS::LakeFormation::DataLakeSettings
    Properties:
      Admins:
        - DataLakePrincipalIdentifier: !Ref DataLakeAdminArn

  # Glue Database
  GlueDatabase:
    Type: AWS::Glue::Database
    Properties:
      CatalogId: !Ref AWS::AccountId
      DatabaseInput:
        Name: !Sub 'data_lake_${Environment}'
        Description: Data lake database for analytics

  # IAM Role for Glue
  GlueServiceRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: glue.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole
      Policies:
        - PolicyName: GlueDataLakePolicy
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - s3:GetObject
                  - s3:PutObject
                  - s3:DeleteObject
                  - s3:ListBucket
                Resource:
                  - !GetAtt RawDataBucket.Arn
                  - !Sub '${RawDataBucket.Arn}/*'
                  - !GetAtt ProcessedDataBucket.Arn
                  - !Sub '${ProcessedDataBucket.Arn}/*'
              - Effect: Allow
                Action:
                  - kms:Decrypt
                  - kms:GenerateDataKey
                Resource: !GetAtt DataLakeKey.Arn
              - Effect: Allow
                Action:
                  - lakeformation:GetDataAccess
                Resource: '*'

  # Glue Crawler for raw data
  RawDataCrawler:
    Type: AWS::Glue::Crawler
    Properties:
      Name: !Sub 'raw-data-crawler-${Environment}'
      Role: !GetAtt GlueServiceRole.Arn
      DatabaseName: !Ref GlueDatabase
      Targets:
        S3Targets:
          - Path: !Sub 's3://${RawDataBucket}/data/'
      SchemaChangePolicy:
        UpdateBehavior: UPDATE_IN_DATABASE
        DeleteBehavior: LOG
      Schedule:
        ScheduleExpression: 'cron(0 2 * * ? *)'

  # Glue ETL Job
  GlueETLJob:
    Type: AWS::Glue::Job
    Properties:
      Name: !Sub 'etl-job-${Environment}'
      Role: !GetAtt GlueServiceRole.Arn
      GlueVersion: '4.0'
      WorkerType: G.1X
      NumberOfWorkers: 2
      Timeout: 60
      DefaultArguments:
        '--job-language': python
        '--enable-metrics': 'true'
        '--enable-continuous-cloudwatch-log': 'true'
        '--enable-spark-ui': 'true'
        '--TempDir': !Sub 's3://${ProcessedDataBucket}/tmp/'
        '--SOURCE_BUCKET': !Ref RawDataBucket
        '--TARGET_BUCKET': !Ref ProcessedDataBucket
      Command:
        Name: glueetl
        ScriptLocation: !Sub 's3://${ProcessedDataBucket}/scripts/etl.py'
        PythonVersion: '3'

  # Athena Workgroup
  AthenaWorkgroup:
    Type: AWS::Athena::WorkGroup
    Properties:
      Name: !Sub 'data-lake-workgroup-${Environment}'
      Description: Workgroup for data lake queries
      WorkGroupConfiguration:
        EnforceWorkGroupConfiguration: true
        ResultConfiguration:
          OutputLocation: !Sub 's3://${AthenaResultsBucket}/results/'
          EncryptionConfiguration:
            EncryptionOption: SSE_KMS
            KmsKey: !Ref DataLakeKey
        BytesScannedCutoffPerQuery: 10737418240  # 10 GB
        PublishCloudWatchMetricsEnabled: true

  # IAM Role for Athena users
  AthenaAnalystRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              AWS: !Sub 'arn:aws:iam::${AWS::AccountId}:root'
            Action: sts:AssumeRole
      Policies:
        - PolicyName: AthenaQueryPolicy
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - athena:StartQueryExecution
                  - athena:GetQueryExecution
                  - athena:GetQueryResults
                  - athena:StopQueryExecution
                  - athena:ListQueryExecutions
                Resource: !Sub 'arn:aws:athena:${AWS::Region}:${AWS::AccountId}:workgroup/${AthenaWorkgroup}'
              - Effect: Allow
                Action:
                  - glue:GetDatabase
                  - glue:GetTable
                  - glue:GetTables
                  - glue:GetPartition
                  - glue:GetPartitions
                Resource:
                  - !Sub 'arn:aws:glue:${AWS::Region}:${AWS::AccountId}:catalog'
                  - !Sub 'arn:aws:glue:${AWS::Region}:${AWS::AccountId}:database/${GlueDatabase}'
                  - !Sub 'arn:aws:glue:${AWS::Region}:${AWS::AccountId}:table/${GlueDatabase}/*'
              - Effect: Allow
                Action:
                  - s3:GetObject
                  - s3:ListBucket
                Resource:
                  - !GetAtt ProcessedDataBucket.Arn
                  - !Sub '${ProcessedDataBucket.Arn}/*'
              - Effect: Allow
                Action:
                  - s3:GetObject
                  - s3:PutObject
                  - s3:ListBucket
                Resource:
                  - !GetAtt AthenaResultsBucket.Arn
                  - !Sub '${AthenaResultsBucket.Arn}/*'
              - Effect: Allow
                Action:
                  - kms:Decrypt
                  - kms:GenerateDataKey
                Resource: !GetAtt DataLakeKey.Arn
              - Effect: Allow
                Action:
                  - lakeformation:GetDataAccess
                Resource: '*'

  # Lake Formation permissions for Glue role
  GlueLakeFormationPermission:
    Type: AWS::LakeFormation::Permissions
    DependsOn: DataLakeSettings
    Properties:
      DataLakePrincipal:
        DataLakePrincipalIdentifier: !GetAtt GlueServiceRole.Arn
      Resource:
        DatabaseResource:
          Name: !Ref GlueDatabase
      Permissions:
        - ALL

  # CloudWatch Alarms
  GlueJobFailureAlarm:
    Type: AWS::CloudWatch::Alarm
    Properties:
      AlarmDescription: Alert when Glue ETL job fails
      MetricName: glue.driver.aggregate.numFailedTasks
      Namespace: Glue
      Statistic: Sum
      Period: 300
      EvaluationPeriods: 1
      Threshold: 1
      ComparisonOperator: GreaterThanOrEqualToThreshold

Outputs:
  RawDataBucketName:
    Description: S3 bucket for raw data ingestion
    Value: !Ref RawDataBucket
    Export:
      Name: !Sub '${AWS::StackName}-RawDataBucket'

  ProcessedDataBucketName:
    Description: S3 bucket for processed/curated data
    Value: !Ref ProcessedDataBucket
    Export:
      Name: !Sub '${AWS::StackName}-ProcessedDataBucket'

  GlueDatabaseName:
    Description: Glue Data Catalog database name
    Value: !Ref GlueDatabase
    Export:
      Name: !Sub '${AWS::StackName}-GlueDatabase'

  AthenaWorkgroupName:
    Description: Athena workgroup for querying data lake
    Value: !Ref AthenaWorkgroup
    Export:
      Name: !Sub '${AWS::StackName}-AthenaWorkgroup'

  AthenaAnalystRoleArn:
    Description: IAM Role ARN for Athena analysts
    Value: !GetAtt AthenaAnalystRole.Arn
    Export:
      Name: !Sub '${AWS::StackName}-AthenaAnalystRole'