AWSTemplateFormatVersion: '2010-09-09'
Transform: AWS::Serverless-2016-10-31
Description: An AWS Serverless Application.
Resources:
  MyLambdaFunction:
    Type: AWS::Serverless::Function
    Properties:
      Handler: src/handlers/myFunction.handler
      Runtime: nodejs14.x
      CodeUri: .
      Events:
        MyApi:
          Type: Api
          Properties:
            Path: /path
            Method: post
  MyDynamoDBTable:
    Type: AWS::Serverless::SimpleTable
    Properties:
      PrimaryKey:
        Name: id
        Type: String
  MyBucket:
    Type: AWS::S3::Bucket

import * as cdk from '@aws-cdk/core';
import * as lambda from '@aws-cdk/aws-lambda';
import * as apigw from '@aws-cdk/aws-apigateway';

export class MyStack extends cdk.Stack {
  constructor(scope: cdk.Construct, id: string, props?: cdk.StackProps) {
    super(scope, id, props);

    // Define the Lambda function
    const myFunction = new lambda.Function(this, 'MyFunction', {
      runtime: lambda.Runtime.NODEJS_14_X,
      handler: 'myFunction.handler',
      code: lambda.Code.fromAsset('lambda'),
    });

    // Define the API Gateway
    const api = new apigw.RestApi(this, 'MyApi');
    const getResource = api.root.addResource('myresource');
    getResource.addMethod('GET', new apigw.LambdaIntegration(myFunction));
  }
}

{
  "name": "knowlo",
  "version": "1.0.0",
  "description": "Monorepo for Knowlo app",
  "private": true,
  "workspaces": [
    "frontend",
    "backend"
  ]
}

Applying project template app for typescript
Initializing a new git repository...
Executing npm install...
npm notice created a lockfile as package-lock.json. You should commit this file.
npm WARN tst@0.1.0 No repository field.
npm WARN tst@0.1.0 No license field.

# Welcome to your CDK TypeScript project!

You should explore the contents of this project. It demonstrates a CDK app with an instance of a stack (`CdkWorkshopStack`)
which contains an Amazon SQS queue that is subscribed to an Amazon SNS topic.

The `cdk.json` file tells the CDK Toolkit how to execute your app.

## Useful commands

 * `npm run build`   compile typescript to js
 * `npm run watch`   watch for changes and compile
 * `npm run test`    perform the jest unit tests
 * `cdk deploy`      deploy this stack to your default AWS account/region
 * `cdk diff`        compare deployed stack with current state
 * `cdk synth`       emits the synthesized CloudFormation template

import { Construct } from 'constructs'
import { CfnOutput, CfnParameter, Duration, RemovalPolicy, Stack, StackProps } from 'aws-cdk-lib'
import * as s3 from 'aws-cdk-lib/aws-s3'
import * as apiGateway from 'aws-cdk-lib/aws-apigateway'
import * as lambda from 'aws-cdk-lib/aws-lambda-nodejs'
import { Runtime } from 'aws-cdk-lib/aws-lambda'
import * as ssm from 'aws-cdk-lib/aws-ssm'
import { RetentionDays } from 'aws-cdk-lib/aws-logs'
import { S3EventSource } from 'aws-cdk-lib/aws-lambda-event-sources'

export class KnowloBackendStack extends Stack {
  constructor(scope: Construct, id: string, props?: StackProps) {
    super(scope, id, props)

    // We'll add AWS resources here!
  }
}

const environmentParameter = new CfnParameter(this, 'Environment', {
  default: 'develop',
  allowedValues: ['develop', 'production'],
  description: 'The application environment'
})

const logLevelParameter = new CfnParameter(this, 'LogLevel', {
  default: 'INFO',
  allowedValues: ['DEBUG', 'INFO', 'WARN', 'ERROR', 'SILENT'],
  description: 'A log level for Lambda functions'
})

const openAiApiKey = new ssm.StringParameter(this, 'OpenAiApiKey', {
  description: 'The OpenAI API Key',
  stringValue: 'change-me'
})

const csvBucket = new s3.Bucket(this, 'CsvBucket', {
  encryption: s3.BucketEncryption.KMS_MANAGED,
  removalPolicy: environmentParameter.valueAsString === 'production' ? RemovalPolicy.RETAIN : RemovalPolicy.DESTROY,
  autoDeleteObjects: environmentParameter.valueAsString === 'develop' ? true : false,
  accessControl: s3.BucketAccessControl.PRIVATE,
})

const api = new apiGateway.RestApi(this, 'AppApi', {})

api.root.addCorsPreflight({
  allowOrigins: ['*'],
  allowMethods: ['OPTIONS', 'GET', 'POST'],
  allowHeaders: ['x-apigateway-header', 'Authorization', 'Content-Type', 'x-amz-date', 'x-amz-security-token'],
  maxAge: Duration.seconds(600),
})

const getUploadUrlFunction = new lambda.NodejsFunction(this, 'GetUploadUrlFunction', {
  entry: './lib/functions/get-upload-url/lambda.ts',
  handler: 'handler',
  runtime: Runtime.NODEJS_18_X,
  timeout: Duration.seconds(30),
  environment: {
    LOG_LEVEL: logLevelParameter.valueAsString,
    CSV_BUCKET: csvBucket.bucketName,
    NODE_OPTIONS: '--enable-source-maps',
  },
  logRetention: environmentParameter.valueAsString === 'production' ? RetentionDays.INFINITE : RetentionDays.ONE_WEEK,
  bundling: {
    sourceMap: true,
  }
})
csvBucket.grantPut(getUploadUrlFunction, 'uploads/*')
csvBucket.grantPut(getUploadUrlFunction, 'status/*')

const getUploadUrlLambdaIntegration = new apiGateway.LambdaIntegration(getUploadUrlFunction, {
  proxy: true,
})

const getUploadUrlApiResource = api.root.addResource('upload')
getUploadUrlApiResource.addMethod('POST', getUploadUrlLambdaIntegration, {})

const processCsvFileFunction = new lambda.NodejsFunction(this, 'ProcessCsvFileFunction', {
  entry: './lib/functions/process-csv-file/lambda.ts',
  handler: 'handler',
  runtime: Runtime.NODEJS_18_X,
  timeout: Duration.minutes(15),
  memorySize: 1024,
  environment: {
    LOG_LEVEL: logLevelParameter.valueAsString,
    CSV_BUCKET: csvBucket.bucketName,
    OPEN_AI_API_KEY: openAiApiKey.stringValue,
    NODE_OPTIONS: '--enable-source-maps',
  },
  logRetention: environmentParameter.valueAsString === 'production' ? RetentionDays.INFINITE : RetentionDays.ONE_WEEK,
  bundling: {
    sourceMap: true,
  }
})
csvBucket.grantRead(processCsvFileFunction, 'uploads/*')
csvBucket.grantWrite(processCsvFileFunction, 'embeddings/*')
csvBucket.grantWrite(processCsvFileFunction, 'status/*')

processCsvFileFunction.addEventSource(new S3EventSource(csvBucket, {
  events: [ s3.EventType.OBJECT_CREATED ],
  filters: [{
    prefix: 'uploads/',
    suffix: '/file.csv',
  }]
}))

new CfnOutput(this, 'ApiEndpoint', {
  value: api.url,
})

.
' README.md
' bin
?   ' knowlo-backend.ts
' cdk.json
' jest.config.js
' lib
?   ' common
?   ?   ' api-gateway-response-generator.ts
?   ?   ' errors
?   ?       ' file-upload.ts
?   ?       ' generate-error.ts
?   ?       ' generic.ts
?   ?       ' types.ts
?   ' functions
?   ?   ' get-upload-url
?   ?   ?   ' lambda.ts
?   ?   ?   ' src
?   ?   ?   ?   ' main.ts
?   ?   ?   ?   ' parser.ts
?   ?   ?   ' types.ts
?   ?   ' process-csv-file
?   ?       ' lambda.ts
?   ?       ' src
?   ?       ?   ' main.ts
?   ?       ?   ' parser.ts
?   ?       ' types.ts
?   ' knowlo-backend-stack.ts
?   ' repositories
?       ' file-repository.ts
?       ' open-ai-repository.ts
' package.json
' test
?   ' backend.test.ts
' tsconfig.json

import { APIGatewayProxyEvent, APIGatewayProxyResult } from 'aws-lambda'
import { Logger } from '@aws-lambda-powertools/logger';
import { httpResponse } from '../../common/api-gateway-response-generator'
import { parseApiEvent } from './src/parser'
import { getUploadUrl } from './src/main'
import { FileRepository } from '../../repositories/file-repository';
import { LogLevel } from '@aws-lambda-powertools/logger/lib/types';
import { generateError } from '../../common/errors/generate-error';
import { ConfigurationError } from '../../common/errors/generic';

export async function handler(event: APIGatewayProxyEvent): Promise<APIGatewayProxyResult> {
  // Create an instance of the Lambda Power Tools Logger
  const logger = new Logger({
    serviceName: 'GetUploadUrlFunction',
    logLevel: process.env.LOG_LEVEL as LogLevel || 'WARN',
  })

  try {
    // Validation
    const bucket = process.env.CSV_BUCKET
    if (!bucket) {
      throw generateError(ConfigurationError)
    }
    // Initialize repositories
    const fileRepository = new FileRepository(bucket)
    // Run the business logic
    const { statusUrl, requestId, uploadUrl } = await getUploadUrl<APIGatewayProxyEvent>({
      event,
      parser: parseApiEvent,
      repositories: {
        fileRepository,
      },
      bucket: bucket,
      region: process.env.AWS_REGION || '',
      logger,
    })
    // Return the response
    return httpResponse({
      body: {
        statusUrl,
        requestId,
        uploadUrl
      },
    })
  } catch (err) {
    // Log an error
    logger.error('Handler error', err as Error)
    // Return an error
    return httpResponse({
      statusCode: 400,
      body: err as Error,
    })
  }
}

import { randomUUID } from 'crypto'
import { PresignedUrlFailed, UnsupportedMimeTypeException } from '../../../common/errors/file-upload'
import { generateError } from '../../../common/errors/generate-error'
import { IGetUploadUrlParams, IGetUploadUrlresponse } from '../types'

// Helper function that should probably go to a separate file
function getFileExtension(mimeType: string): string {
  switch (mimeType) {
    case 'text/csv':
      return 'csv'

    default:
      throw generateError(UnsupportedMimeTypeException)
  }
}

// The main business logic
// We use TypeScript generics because we want to be able to test it
// in isolation and without mocking AWS event
export async function getUploadUrl<T>({
  event, parser, repositories, logger
}: IGetUploadUrlParams<T>): Promise<IGetUploadUrlresponse> {
  try {
    // Get the file extension
    const { mimeType } = parser(event)
    const extension = getFileExtension(mimeType)
    // Generate the UUID
    const requestId = randomUUID()
    // Prepare file paths (in S3 but our business logic don't care about the actual service)
    const uploadFileName = `uploads/${requestId}/file.${extension}`
    const statusFileName = `status/${requestId}.json`
    // Create the file using fileRepository
    await repositories.fileRepository.createFile(statusFileName, JSON.stringify({
      requestId,
      status: 'IN_PROGRESS',
    }))
    // Generate presinged URLs for uploading a file and checking the status
    const uploadUrl = await repositories.fileRepository.getPresignedUploadUrl(uploadFileName)
    const statusUrl = await repositories.fileRepository.getPresignedUploadUrl(statusFileName)

    // Return the result
    return {
      uploadUrl,
      statusUrl,
      requestId,
    }
  } catch (error) {
    logger.error('getUploadUrl error', error as Error)
    throw generateError(PresignedUrlFailed)
  }
}

import { APIGatewayProxyEvent } from 'aws-lambda'
import { IParsedEvent } from '../types'

export function parseApiEvent(event: APIGatewayProxyEvent): IParsedEvent {
  // Parse the API Gateway proxy event
  const mimeType = event.queryStringParameters?.type || 'text/csv'
  // Return the value
  return {
    mimeType,
  }
}

import { S3Event } from 'aws-lambda'
import { Logger } from '@aws-lambda-powertools/logger'
import { LogLevel } from '@aws-lambda-powertools/logger/lib/types/Log'
import { FileRepository } from '../../repositories/file-repository';
import { generateError } from '../../common/errors/generate-error';
import { ConfigurationError } from '../../common/errors/generic';
import { processCsvFile } from './src/main'
import { parseS3Event } from './src/parser';
import { OpenAiRepository } from '../../repositories/open-ai-repository';

export async function handler(event: S3Event): Promise<void> {
  const logger = new Logger({
    serviceName: 'GetUploadUrlFunction',
    logLevel: process.env.LOG_LEVEL as LogLevel || 'WARN',
  })

  const bucket = process.env.CSV_BUCKET
  if (!bucket) {
    throw generateError(ConfigurationError)
  }
  
  const openAiApiKey = process.env.OPEN_AI_API_KEY
  if (!openAiApiKey) {
    throw generateError(ConfigurationError)
  }
  // We have 2 repositories now
  const fileRepository = new FileRepository(bucket, logger)
  const aiRepository = new OpenAiRepository(openAiApiKey, logger)
  await processCsvFile<S3Event>({
    event,
    parser: parseS3Event,
    logger,
    repositories: {
      fileRepository,
      aiRepository,
    },
    bucket,
    region: process.env.AWS_REGION || '',
  })

  logger.info(JSON.stringify(event, null, 2))
}

import { PresignedUrlFailed } from '../../../common/errors/file-upload'
import { generateError } from '../../../common/errors/generate-error'
import { ICsvRecords, IGetUploadUrlParams } from '../types'
import { parse } from 'csv-parse/sync'
// Helper function
function getUUID(input: string): string | null {
  const uuidRegex = /uploads\/([a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})\/file\.csv/;
  const match = input.match(uuidRegex);
  return match ? match[1] : null;
}
// Business logic
export async function processCsvFile<T>({
  event, parser, repositories, logger
}: IGetUploadUrlParams<T>): Promise<void> {
  try {
    const files = parser(event)
    // There can be multiple files, we'll do the same for each of them
    await Promise.all(files.map(async file => {
      // Get a file
      repositories.fileRepository.setBucket(file.bucket)
      const csvFile = await repositories.fileRepository.getFile(file.filePath)
      logger.debug('CSV File', csvFile)
      // Parse CSV and filter out unpublished Helpdesk articles,
      // this part will change as it's too specific for Crisp chat at the moment
      const records: ICsvRecords[] = parse(csvFile, {
        columns: true,
        skip_empty_lines: true,
      })
      const publishedRecords = records.filter(rec => rec['published_at'] !== '')
      
      // Create embeddings for each Helpdesk article
      const output = await Promise.all(
        publishedRecords.map(async rec => {
          const embedding = await repositories.aiRepository.createEmbeddings(JSON.stringify(rec))
          return {
            ...rec,
            embedding,
          }
        })
      )

      logger.debug('Embeddings output', JSON.stringify(output, null, 2))
      const requestId = getUUID(file.filePath)
      logger.info(`Request ID: ${requestId}`)
      if (requestId) {
        // Save embeddings to the S3 bucket
        const embeddingsFilePath = `embeddings/${requestId}/embeddings.json`
        const statusFilePath = `status/${requestId}.json`
        logger.info(`Embeddings URL: ${embeddingsFilePath}`)
        await repositories.fileRepository.createFile(embeddingsFilePath, JSON.stringify(output, null, 2))
        // And update the request status
        await repositories.fileRepository.createFile(statusFilePath, JSON.stringify({
          requestId,
          status: 'DONE',
        }))
      }
    }))
  } catch (error) {
    logger.error('getUploadUrl error', error as Error)
    throw generateError(PresignedUrlFailed)
  }
}

import { Logger } from '@aws-lambda-powertools/logger'
import { S3Client, PutObjectCommand, PutObjectCommandOutput, GetObjectCommand, GetObjectCommandOutput } from '@aws-sdk/client-s3'
import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
import { generateError } from '../common/errors/generate-error'
import { FileNotFound } from '../common/errors/file-upload'

export class FileRepository {
  bucket: string
  region: string
  s3Client: S3Client
  logger: Logger

  constructor(bucket: string, logger?: Logger) {
    this.bucket = bucket
    this.region = process.env.AWS_REGION || 'eu-central-1'
    this.s3Client = new S3Client({})
    this.logger = logger || new Logger()
  }

  setBucket(bucketName: string): void {
    this.bucket = bucketName
  }

  async getPresignedUploadUrl(fileName: string, expiresIn: number = 900): Promise<string> {
    const command = new PutObjectCommand({
      Bucket: this.bucket,
      Key: fileName,
      ACL: 'private',
    })
    return getSignedUrl(this.s3Client, command, { expiresIn })
  }

  async getPresignedReadUrl(fileName: string, expiresIn: number = 900): Promise<string> {
    const command = new GetObjectCommand({
      Bucket: this.bucket,
      Key: fileName,
    })
    return getSignedUrl(this.s3Client, command, { expiresIn })
  }

  async createFile(fileName: string, fileContent: string | Buffer): Promise<void> {
    const command = new PutObjectCommand({
      Bucket: this.bucket,
      Key: fileName,
      ACL: 'private',
      Body: fileContent,
    })
    const response = await this.s3Client.send(command)
    this.logger.debug('FileRepository.createFile > S3 PutObjectCommand Output', JSON.stringify(response))
  }

  async getFile(fileName: string): Promise<string> {
    const command = new GetObjectCommand({
      Bucket: this.bucket,
      Key: fileName,
    })
    const response = await this.s3Client.send(command)
    if (!response.Body) {
      throw generateError(FileNotFound)
    }
    return await response.Body.transformToString()
  }
}

import { Logger } from '@aws-lambda-powertools/logger'

export class OpenAiRepository {
  apiKey: string
  logger: Logger

  constructor(apiKey: string, logger?: Logger) {
    this.apiKey = apiKey
    this.logger = logger || new Logger()
  }

  async createEmbeddings(input: string, model = 'text-embedding-ada-002'): Promise<any> {
    try {
      const rawResponse = await fetch('https://api.openai.com/v1/embeddings', {
        method: 'POST',
        headers: {
          Authorization: `Bearer ${this.apiKey}`,
          'Content-Type': 'application/json',
        },
        body: JSON.stringify({
          input,
          model
        })
      })
      const response = await rawResponse.json()
      this.logger.debug('OpenAI usage', response.usage)
      return response
    } catch (err) {
      this.logger.error('OpenAiRepository.createEmbeddings > Error', err as Error)
      throw err
    }
  }
}

{
  "name": "backend",
  "version": "0.1.0",
  "bin": {
    "knowlo-backend": "bin/knowlo-backend.js"
  },
  "scripts": {
    "build": "tsc",
    "watch": "tsc -w",
    "test": "jest",
    "cdk": "cdk"
  },
  "devDependencies": {
    "@types/jest": "^29.4.0",
    "@types/node": "18.14.6",
    "aws-cdk": "2.78.0",
    "esbuild": "^0.17.18",
    "jest": "^29.5.0",
    "ts-jest": "^29.0.5",
    "ts-node": "^10.9.1",
    "typescript": "~4.9.5"
  },
  "dependencies": {
    "@aws-lambda-powertools/logger": "^1.8.0",
    "@aws-sdk/client-s3": "^3.331.0",
    "@aws-sdk/s3-request-presigner": "^3.331.0",
    "@middy/core": "^4.4.1",
    "@types/aws-lambda": "^8.10.115",
    "@types/csv-parse": "^1.2.2",
    "aws-cdk-lib": "2.78.0",
    "aws-lambda": "^1.0.7",
    "constructs": "^10.0.0",
    "csv-parse": "^5.3.10"
  }
}

npm run cdk deploy -- --parameters LogLevel=DEBUG

?  KnowloBackendStack (no changes)

?  Deployment time: 2.89s

Outputs:
KnowloBackendStack.ApiEndpoint = https://srxxxxxxr0.execute-api.eu-west-1.amazonaws.com/prod/
KnowloBackendStack.AppApiEndpoint979256A8 = https://srxxxxxxr0.execute-api.eu-west-1.amazonaws.com/prod/
Stack ARN:
arn:aws:cloudformation:eu-west-1:xxxxxxxxxxxx:stack/KnowloBackendStack/cbxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxx8b

?  Total time: 6.58s