Da MariaDB a Parquet con Node.js e Apache Arrow

In questo articolo vediamo come creare un file Parquet con Node.js e Apache Arrow.

I dati li prenderemo da un database MariaDB.

Possiamo installare queste dipendenze direttamente con npm:

npm install mysql2 parquetjs

Qui sotto un pò di codice:

const mysql = require('mysql2/promise');
const parquet = require('parquetjs');

const dbConfig = {
    host: 'localhost',
    user: 'root',
    password: '9211',
    database: 'Sql592686_5'
};

const outputFile = 'movimenti.parquet';

async function exportToParquet() {
    let connection;

    try {
        connection = await mysql.createConnection(dbConfig);

        const schema = new parquet.ParquetSchema({
            mov_tipo: {type: 'UTF8'},
            mov_valore: {type: 'DOUBLE'},
            mov_data: {type: 'UTF8'},
            causale_nome: {type: 'UTF8'}
        });

        const writer = await parquet.ParquetWriter.openFile(schema, outputFile, {
            compression: 'SNAPPY'
        });

        const query = `SELECT mov_tipo,
                                     mov_valore,
                                     mov_data,
                                     causale_nome
                              FROM movimenti
                                  INNER JOIN causali ON causale_id = mov_causale_fk`;

        const [rows] = await connection.execute(query);

        let recordCount = 0;

        for (const row of rows) {
            await writer.appendRow({
                mov_tipo: row.mov_tipo,
                mov_valore: row.mov_valore,
                mov_data: row.mov_data.toISOString().split('T')[0],
                causale_nome: row.causale_nome
            });

            recordCount++;

            if (recordCount % 1000 === 0) {
                console.log(`Scritti ${recordCount} record...`);
            }
        }

        await writer.close();

        console.log(`Totale record esportati: ${recordCount}`);

    } catch (error) {
        console.error('Errore:', error.message);
    } finally {
        if (connection) {
            connection.end();
        }
    }
}

exportToParquet();

Enjoy!