serverless.yml

org: beloinvres
app: amazon-scraper
service: pipeline

configValidationMode: error

provider:
  name: aws
  runtime: python3.8
  stackName: ${self:custom.Name}-stack
  deploymentBucket:
    name: ${self:custom.Name}-deploy
  stackTags:
    PROJECT: AmazonScraper
    CLIENT: dk
  iam:
    role:
      name: ${self:custom.Name}-role
      statements:
        - Effect: Allow
          Action:
            - s3:ListBucket
          Resource:
            - 'arn:aws:s3:::${self:custom.Bucket}'
        - Effect: Allow
          Action:
            - s3:PutObject
            - s3:GetObject
            - s3:DeleteObject
          Resource:
            - 'arn:aws:s3:::${self:custom.Bucket}/*'
        - Effect: Allow
          Action:
            - secretsmanager:GetSecretValue
          Resource:
            - 'arn:aws:secretsmanager:${aws:region}:${aws:accountId}:secret:${self:custom.DBSecretId}*'
        - Effect: Allow
          Action:
            - sqs:GetQueueUrl
          Resource:
            - '*'
        - Effect: Allow
          Action:
            - sqs:ReceiveMessage
            - sqs:SendMessage
          Resource:
            - 'arn:aws:sqs:${aws:region}:${aws:accountId}:amazon-*'

package:
  patterns:
    - '!.git/**'
    - '!.env'
    - '!node_modules/**'
    - '!env/**'
    - '!Data/**'
    - '!env_amazon_pipeline/**'

functions:
  01-parse-products-list:
    name: ${self:custom.Name}-01-parse-products-list
    handler: parse_products_list.handler
    timeout: 900
    environment:
      DB_SECRET_ID: ${self:custom.DBSecretId}
    events:
      - s3:
          bucket: ${self:custom.Bucket}
          event: s3:ObjectCreated:*
          rules:
            - prefix: products-list/
            - suffix: .xlsx
          existing: true
    layers:
      - Ref: PythonRequirementsLambdaLayer

  02-request-variations:
    name: ${self:custom.Name}-02-request-variations
    handler: request_variations.handler
    timeout: 900
    memorySize: 5120
    environment:
      DB_SECRET_ID: ${self:custom.DBSecretId}
      BUCKET: ${self:custom.Bucket}
      QUEUE: amazon-variation-pages-queue.fifo
    events:
      - schedule: rate(8 hours)
    layers:
      - Ref: PythonRequirementsLambdaLayer

  03-parse-new-asins:
    name: ${self:custom.Name}-03-parse-new-asins
    handler: parse_new_asins.handler
    timeout: 900
    memorySize: 5120
    environment:
      DB_SECRET_ID: ${self:custom.DBSecretId}
    events:
      - s3:
          bucket: ${self:custom.Bucket}
          event: s3:ObjectCreated:*
          rules:
            - prefix: html/variations/
            - suffix: .html
          existing: true
    layers:
      - Ref: PythonRequirementsLambdaLayer

  04-parse-metrics:
    name: ${self:custom.Name}-04-parse-metrics
    handler: parse_metrics.handler
    timeout: 900
    memorySize: 5120
    environment:
      DB_SECRET_ID: ${self:custom.DBSecretId}
      BUCKET: ${self:custom.Bucket}
      N_MAX_FILES: 500
    events:
      - schedule: rate(30 minutes)
    layers:
      - Ref: PythonRequirementsLambdaLayer

  05-parse-best-sellers-ranking:
    name: ${self:custom.Name}-05-parse-best-sellers-ranking
    handler: parse_best_sellers_ranking.handler
    timeout: 900
    memorySize: 5120
    environment:
      DB_SECRET_ID: ${self:custom.DBSecretId}
      BUCKET: ${self:custom.Bucket}
      N_MAX_FILES: 500
    events:
      - schedule: rate(30 minutes)
    layers:
      - Ref: PythonRequirementsLambdaLayer

  06-parse-frequently-bought-together:
    name: ${self:custom.Name}-06-parse-frequently-bought-together
    handler: parse_frequently_bought_together.handler
    timeout: 900
    memorySize: 5120
    environment:
      DB_SECRET_ID: ${self:custom.DBSecretId}
      BUCKET: ${self:custom.Bucket}
      N_MAX_FILES: 500
    events:
      - schedule: rate(30 minutes)
    layers:
      - Ref: PythonRequirementsLambdaLayer

  07-parse-similar-products:
    name: ${self:custom.Name}-07-parse-similar-products
    handler: parse_similar_products.handler
    timeout: 900
    memorySize: 5120
    environment:
      DB_SECRET_ID: ${self:custom.DBSecretId}
      BUCKET: ${self:custom.Bucket}
      N_MAX_FILES: 500
    events:
      - schedule: rate(30 minutes)
    layers:
      - Ref: PythonRequirementsLambdaLayer

  08-request-categories:
    name: ${self:custom.Name}-08-request-categories
    handler: request_categories.handler
    timeout: 900
    memorySize: 5120
    environment:
      DB_SECRET_ID: ${self:custom.DBSecretId}
      BUCKET: ${self:custom.Bucket}
      QUEUE: amazon-category-pages-queue.fifo
    events:
      - schedule: rate(8 hours)
    layers:
      - Ref: PythonRequirementsLambdaLayer

  09-parse-featured-products:
    name: ${self:custom.Name}-09-parse-featured-products
    handler: parse_featured_products.handler
    timeout: 900
    memorySize: 5120
    environment:
      DB_SECRET_ID: ${self:custom.DBSecretId}
      BUCKET: ${self:custom.Bucket}
      N_MAX_FILES: 500
    events:
      - schedule: rate(30 minutes)
    layers:
      - Ref: PythonRequirementsLambdaLayer

  11-deliver-variations:
    name: ${self:custom.Name}-11-deliver-variations
    handler: deliver_variations.handler
    timeout: 900
    memorySize: 5120
    environment:
      DB_SECRET_ID: ${self:custom.DBSecretId}
      BUCKET: ${self:custom.Bucket}
    events:
      - schedule: rate(2 hours)
    layers:
      - Ref: PythonRequirementsLambdaLayer

  12-deliver-categories:
    name: ${self:custom.Name}-12-deliver-categories
    handler: deliver_categories.handler
    timeout: 900
    memorySize: 5120
    environment:
      DB_SECRET_ID: ${self:custom.DBSecretId}
      BUCKET: ${self:custom.Bucket}
    events:
      - schedule: rate(2 hours)
    layers:
      - Ref: PythonRequirementsLambdaLayer


plugins:
  - serverless-deployment-bucket
  - serverless-plugin-resource-tagging
  - serverless-python-requirements


custom:
  Name: ${self:app}-${self:service}-${opt:stage, 'dev'}
  Bucket: ${self:app}-data
  DBSecretId: ${self:app}-redshift-credentials

  pythonRequirements:
    fileName: requirements.txt
    dockerizePip: true
    layer: true
    useStaticCache: true