Convert GeoJSON to PMTiles with AWS Serverless Pattern

import * as cdk from 'aws-cdk-lib';
import { Construct } from 'constructs';
import * as s3 from 'aws-cdk-lib/aws-s3';
import * as lambda from 'aws-cdk-lib/aws-lambda';
import * as s3n from 'aws-cdk-lib/aws-s3-notifications';
import * as path from 'path';

export class AutoTilerStack extends cdk.Stack {
    constructor(scope: Construct, id: string, props?: cdk.StackProps) {
        super(scope, id, props);

        // S3 buckets for input GeoJSON and output PMTiles
        const geojsonBucket = new s3.Bucket(this, 'GeojsonBucket', {
            removalPolicy: cdk.RemovalPolicy.DESTROY,
            autoDeleteObjects: true,
        });

        const pmtilesBucket = new s3.Bucket(this, 'PmtilesBucket', {
            removalPolicy: cdk.RemovalPolicy.DESTROY,
            autoDeleteObjects: true,
        });

        // Lambda function to run tippecanoe for conversion
        const tippecanoeLambda = new lambda.DockerImageFunction(
            this,
            'TippecanoeLambda',
            {
                code: lambda.DockerImageCode.fromImageAsset(
                    path.join(__dirname, '../lambda'),
                ),
                timeout: cdk.Duration.minutes(15),
                memorySize: 1770, // 2 vCPUs
                environment: {
                    OUTPUT_BUCKET: pmtilesBucket.bucketName,
                },
                architecture: lambda.Architecture.ARM_64,
            },
        );

        // Grant permissions
        geojsonBucket.grantRead(tippecanoeLambda);
        pmtilesBucket.grantReadWrite(tippecanoeLambda);

        // Configure S3 event notification to trigger Lambda directly
        geojsonBucket.addEventNotification(
            s3.EventType.OBJECT_CREATED,
            new s3n.LambdaDestination(tippecanoeLambda),
            { suffix: '.geojson' }, // Only trigger for .geojson files
        );

        // Output the bucket names
        new cdk.CfnOutput(this, 'GeojsonBucketName', {
            value: geojsonBucket.bucketName,
            description: 'Name of the S3 bucket for GeoJSON files',
        });

        new cdk.CfnOutput(this, 'PMTilesBucketName', {
            value: pmtilesBucket.bucketName,
            description: 'Name of the S3 bucket for PMTiles files',
        });

        new cdk.CfnOutput(this, 'PMTilesBucketWebsiteUrl', {
            value: pmtilesBucket.bucketWebsiteUrl,
            description: 'URL of the PMTiles bucket website',
        });
    }
}

import * as AWS from 'aws-sdk';
import { S3Event, S3EventRecord } from 'aws-lambda';
import { spawn, ChildProcess } from 'child_process';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';

const s3 = new AWS.S3();

export const handler = async (event: S3Event): Promise<any> => {
    console.log('Received event:', JSON.stringify(event, null, 2));

    // Process only the first record (we expect only one per invocation)
    const record: S3EventRecord = event.Records[0];

    // Get the S3 bucket and key from the event
    const srcBucket = record.s3.bucket.name;
    const srcKey = decodeURIComponent(record.s3.object.key.replace(/\+/g, ' '));

    // Get the output bucket from environment variables
    const dstBucket = process.env.OUTPUT_BUCKET as string;

    // Create temporary directory for processing
    const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'geojson-'));
    const localGeojsonPath = path.join(tmpDir, 'input.geojson');
    const outputDir = path.join(tmpDir, 'output');

    try {
        // Create output directory
        fs.mkdirSync(outputDir, { recursive: true });

        // Download the GeoJSON file from S3
        console.log(`Downloading ${srcKey} from ${srcBucket}`);
        const geojsonObject = await s3
            .getObject({
                Bucket: srcBucket,
                Key: srcKey,
            })
            .promise();

        // Write the GeoJSON to a local file
        fs.writeFileSync(localGeojsonPath, geojsonObject.Body as Buffer);

        // Extract the base name without extension for the tile directory
        const baseName = path.basename(srcKey, path.extname(srcKey));

        // Run tippecanoe to convert GeoJSON to PMTiles
        console.log('Running tippecanoe');
        await runTippecanoe(localGeojsonPath, outputDir, baseName);

        // Upload the PMTiles file to S3
        console.log('Uploading PMTiles to S3');
        const pmtilesPath = path.join(outputDir, `${baseName}.pmtiles`);
        await uploadPMTiles(pmtilesPath, dstBucket, '');

        return {
            statusCode: 200,
            body: JSON.stringify({
                message: 'GeoJSON successfully converted to PMTiles',
                source: srcKey,
                destination: `${baseName}.pmtiles`,
            }),
        };
    } catch (error) {
        console.error('Error:', error);
        throw error;
    } finally {
        // Clean up temporary files
        try {
            fs.rmSync(tmpDir, { recursive: true, force: true });
        } catch (err) {
            console.error('Error cleaning up temporary files:', err);
        }
    }
};

// Function to run tippecanoe
async function runTippecanoe(
    inputFile: string,
    outputDir: string,
    baseName: string,
): Promise<void> {
    return new Promise((resolve, reject) => {
        const tileDir = path.join(outputDir, baseName);
        fs.mkdirSync(tileDir, { recursive: true });

        const tippecanoe = spawn('tippecanoe', [
            '-o',
            `${tileDir}.pmtiles`,
            '-z',
            '14', // Maximum zoom level
            '-M',
            '1000000', // max filesize of each tile
            '-l',
            baseName, // Layer name
            inputFile,
        ]);

        setupProcessListeners(tippecanoe, 'tippecanoe', (code) => {
            if (code !== 0) {
                reject(new Error(`tippecanoe process exited with code ${code}`));
                return;
            }

            resolve();
        });
    });
}

// Helper function to set up process listeners
function setupProcessListeners(
    process: ChildProcess,
    name: string,
    onClose: (code: number | null) => void,
): void {
    process.stdout?.on('data', (data) => {
        console.log(`${name} stdout: ${data}`);
    });

    process.stderr?.on('data', (data) => {
        console.error(`${name} stderr: ${data}`);
    });

    process.on('close', onClose);
}

// Function to upload a PMTiles file to S3
async function uploadPMTiles(
    pmtilesPath: string,
    bucket: string,
    prefix: string,
): Promise<void> {
    const fileContent = fs.readFileSync(pmtilesPath);
    const fileName = path.basename(pmtilesPath);
    // Use the same path format as in the response
    const s3Key = prefix === '' ? fileName : `${prefix}/${fileName}`;

    await s3
        .putObject({
            Bucket: bucket,
            Key: s3Key,
            Body: fileContent,
            ContentType: 'application/octet-stream',
        })
        .promise();

    console.log(`Uploaded ${s3Key} to ${bucket}`);
}

FROM public.ecr.aws/lambda/nodejs:18

# Install dependencies for tippecanoe
RUN yum update -y && \
    yum install -y gcc-c++ make git sqlite-devel zlib-devel

# Clone and build tippecanoe
RUN git clone https://github.com/felt/tippecanoe.git && \
    cd tippecanoe && \
    make -j && \
    make install

# No need for mb-util since we're using PMTiles directly

# Copy package.json and install dependencies
COPY package.json tsconfig.json ${LAMBDA_TASK_ROOT}/
RUN npm install

# Copy TypeScript source code
COPY app.ts ${LAMBDA_TASK_ROOT}/

# Build TypeScript code
RUN npm run build

# Set the CMD to your handler
CMD [ "dist/app.handler" ]