diff --git a/custom/ExportCsv.vue b/custom/ExportCsv.vue index 99ac54d..9ccebcf 100644 --- a/custom/ExportCsv.vue +++ b/custom/ExportCsv.vue @@ -48,6 +48,7 @@ const open = ref(false); const rootRef = ref(null); const allCount = ref(null); const filteredCount = ref(null); +let countsPromise: Promise | null = null; defineExpose({ click: () => { toggle(); }, @@ -91,7 +92,7 @@ onUnmounted(() => document.removeEventListener('mousedown', handleClickOutside)) function toggle() { open.value = !open.value; if (open.value) { - fetchCounts(); + countsPromise = fetchCounts(); } } @@ -119,7 +120,55 @@ async function fetchCounts() { function run(select: string) { open.value = false; - exportCsv(select); + if (props.meta?.exportBigDataset) { + startExportJob(select); + } else { + exportCsv(select); + } +} + +/** + * Total is taken from counts already fetched for the dropdown labels, so the backend does not have + * to run an extra count query just to render export progress. + */ +async function getTotalForSelection(select: string): Promise { + if (select === 'selected') { + return props.checkboxes?.length || 0; + } + await countsPromise; + const count = select === 'filtered' ? filteredCount.value : allCount.value; + return count ?? undefined; +} + +async function startExportJob(select: string) { + inProgress.value = true; + try { + const resp = await callAdminForthApi({ + path: `/plugin/${props.meta?.pluginInstanceId}/start-export-job`, + method: 'POST', + body: { + filters: select === 'filtered' ? filtersStore.getFilters() : [], + sort: filtersStore.getSort(), + selectedIds: select === 'selected' ? props.checkboxes : undefined, + totalRows: await getTotalForSelection(select), + }, + }); + + if (!resp?.ok) { + throw new Error(resp?.error || t('Failed to start export')); + } + + // registered globally by the background jobs plugin + (window as any).OpenJobInfoPopup?.(resp.jobId); + } catch (error) { + adminforth.alert({ + message: error instanceof Error ? error.message : t('Export failed'), + variant: 'danger', + }); + } finally { + inProgress.value = false; + adminforth.list.closeThreeDotsDropdown(); + } } function downloadFile(data: string, filename: string) { diff --git a/custom/ExportCsvJobViewComponent.vue b/custom/ExportCsvJobViewComponent.vue new file mode 100644 index 0000000..9903478 --- /dev/null +++ b/custom/ExportCsvJobViewComponent.vue @@ -0,0 +1,100 @@ + + + diff --git a/exportMultipartUpload.ts b/exportMultipartUpload.ts new file mode 100644 index 0000000..be1072e --- /dev/null +++ b/exportMultipartUpload.ts @@ -0,0 +1,325 @@ +import { AdminForthDataTypes, AdminForthSortDirections, Filters } from 'adminforth'; +import type { AdminForthResourceColumn, AdminUser, IAdminForthSort } from 'adminforth'; +import { stringify } from 'csv/sync'; +import type BackgroundJobsPlugin from '@adminforth/background-jobs'; +import type ImportExportPlugin from './index.js'; + +export const EXPORT_CSV_JOB_HANDLER_NAME = 'export_csv_job_handler'; +export const MINIMAL_BUFFER_SIZE_MB = 5; + +/** How many records are pulled from the database per iteration, unless overridden in plugin options. */ +export const DEFAULT_READ_CHUNK_SIZE = 100; +/** Minimal delay between two progress publications, to not spam websocket/db on fast datasets. */ +const PROGRESS_PUBLISH_INTERVAL_MS = 1000; +/** Minimal delay between two checks whether the job was cancelled from UI. */ +const CANCELLATION_CHECK_INTERVAL_MS = 2000; +/** Lifetime of the presigned download link handed to the browser. */ +const DOWNLOAD_URL_EXPIRES_IN_SECONDS = 3600; + +type TaskState = { + filters: any; + sort: IAdminForthSort[]; + fileKey: string; + fileName: string; +}; + +type TaskHandlerParams = { + jobId: string; + getState: () => Promise>; +}; + +/** + * ✅Columns which end up in the exported file. Mirrors {@link ImportExportPlugin.exportCsv}. + */ +function getExportColumns(plugin: ImportExportPlugin): { + columns: AdminForthResourceColumn[]; + fields: string[]; + columnsToForceQuote: boolean[]; +} { + const columns = plugin.resourceConfig.columns.filter((col) => !col.virtual && !col.backendOnly); + return { + columns, + fields: columns.map((col) => col.name), + // numbers and booleans are left bare, everything else is quoted: + // quoting all columns breaks BI/Excel tasks + columnsToForceQuote: columns.map((col) => ( + col.type !== AdminForthDataTypes.FLOAT + && col.type !== AdminForthDataTypes.INTEGER + && col.type !== AdminForthDataTypes.BOOLEAN + && col.type !== AdminForthDataTypes.DECIMAL + )), + }; +} + +/** + * ✅Turns a record into an array of CSV cells. `null` is returned for empty cells so that + * csv-stringify renders them as an empty unquoted value. + */ +function serializeRow(columns: AdminForthResourceColumn[], row: Record): (string | null)[] { + return columns.map((col) => { + const value = row[col.name]; + if (value === null || value === undefined) { + return null; + } + if (col.type === AdminForthDataTypes.JSON || col.isArray?.enabled) { + return JSON.stringify(value); + } + if (value instanceof Date) { + return value.toISOString(); + } + return `${value}`; + }); +} + +/** + * ✅All cells are already strings, so the only thing `cast` does here is deciding per-column + * whether the cell has to be quoted. + */ +function buildCsvChunk(rows: (string | null)[][], columnsToForceQuote: boolean[]): string { + return stringify(rows, { + quote: '"', + escape: '"', + cast: { + string: (value: string, context: { index: number }) => ( + columnsToForceQuote[context.index] + ? { value, quoted: true, quoted_empty: true } + : value + ), + }, + }); +} +//✅ +function getBackgroundJobsPlugin(plugin: ImportExportPlugin): BackgroundJobsPlugin { + const backgroundJobsPlugin = plugin.adminforth.getPluginByClassName('BackgroundJobsPlugin'); + if (!backgroundJobsPlugin) { + throw new Error('BackgroundJobsPlugin is required for big dataset export, please add it to your plugins'); + } + return backgroundJobsPlugin; +} + +//✅ +function getJobsResourceMeta(backgroundJobsPlugin: BackgroundJobsPlugin): { resourceId: string; pkColumn: string } { + const resourceConfig = backgroundJobsPlugin.resourceConfig; + return { + resourceId: resourceConfig.resourceId, + pkColumn: resourceConfig.columns.find((col) => col.primaryKey).name, + }; +} + +//✅ +async function getJobRecord(plugin: ImportExportPlugin, jobId: string): Promise | null> { + const backgroundJobsPlugin = getBackgroundJobsPlugin(plugin); + const { resourceId, pkColumn } = getJobsResourceMeta(backgroundJobsPlugin); + return await plugin.adminforth.resource(resourceId).get(Filters.EQ(pkColumn, jobId)); +} + +/** + * ✅Pagination by limit/offset is only stable when the sort is unambiguous, otherwise the database + * is free to return the same record in two different chunks. Appending the primary key fixes it. + */ +function buildStableSort(plugin: ImportExportPlugin, sort: IAdminForthSort[]): IAdminForthSort[] { + const primaryKeyColumn = plugin.resourceConfig.columns.find((col) => col.primaryKey); + const normalizedSort: IAdminForthSort[] = Array.isArray(sort) ? [...sort] : []; + if (!primaryKeyColumn || normalizedSort.some((s) => s.field === primaryKeyColumn.name)) { + return normalizedSort; + } + return [...normalizedSort, { field: primaryKeyColumn.name, direction: AdminForthSortDirections.asc }]; +} + +/** + * ✅Progress of the standard background-jobs UI is derived from the number of finished tasks, and + * the whole export is a single task. So the percentage is published manually here. + */ +async function publishProgress(plugin: ImportExportPlugin, jobId: string, percent: number): Promise { + try { + const backgroundJobsPlugin = getBackgroundJobsPlugin(plugin); + const { resourceId } = getJobsResourceMeta(backgroundJobsPlugin); + await plugin.adminforth.resource(resourceId).update(jobId, { + [backgroundJobsPlugin.options.progressField]: percent, + }); + plugin.adminforth.websocket.publish('/background-jobs-job-update', { jobId, progress: percent }); + } catch (e) { + console.error(`ImportExport: failed to publish progress for export job ${jobId}:`, e); + } +} + +//✅ +async function isJobCancelled(plugin: ImportExportPlugin, jobId: string): Promise { + try { + const backgroundJobsPlugin = getBackgroundJobsPlugin(plugin); + const jobRecord = await getJobRecord(plugin, jobId); + return jobRecord?.[backgroundJobsPlugin.options.statusField] === 'CANCELLED'; + } catch (e) { + console.error(`ImportExport: failed to read status of export job ${jobId}:`, e); + return false; + } +} + +//✅ +async function abortQuietly(writer: { abort: () => Promise }, fileKey: string): Promise { + try { + await writer.abort(); + } catch (e) { + console.error(`ImportExport: failed to abort export upload "${fileKey}":`, e); + } +} + +//✅ +export async function runExportCsvJob( + plugin: ImportExportPlugin, + { jobId, getState }: TaskHandlerParams, +): Promise { + const backgroundJobsPlugin = getBackgroundJobsPlugin(plugin); + const { filters, sort, fileKey } = (await getState()) as TaskState; + const { storageAdapter, bufferSizeMb, readChunkSize } = plugin.options.exportBigDataset; + const chunkSize = readChunkSize ?? DEFAULT_READ_CHUNK_SIZE; + + const connector = plugin.adminforth.connectors[plugin.resourceConfig.dataSource]; + //normalize filters and sort + const normalizedFilters = connector.validateAndNormalizeInputFilters(filters); + const stableSort = buildStableSort(plugin, sort); + //get columns, fields in format like: ['id', 'name', 'email', 'balance'] and columnsToForceQuote in format like: [false, true, true, false] + const { columns, fields, columnsToForceQuote } = getExportColumns(plugin); + + const estimatedTotal: number = (await backgroundJobsPlugin.getJobStateField(jobId, 'totalRows')) ?? 0; + + const writer = await storageAdapter.createWriteStream(fileKey, 'text/csv', bufferSizeMb); + + let exportedRows = 0; + let lastProgressPublishedAt = 0; + let lastCancellationCheckAt = Date.now(); + let cancelled = false; + + try { + // BOM keeps Excel happy with non-ASCII values + // Add this symbol to the beginning of the file to indicate that it is UTF-8 encoded. (requred for some versions of Excel for some reason. Without this symbol encoding can be broken) + await writer.write('' + buildCsvChunk([fields], columnsToForceQuote)); + for (let offset = 0; ; offset += chunkSize) { + const { data } = await connector.getData({ + resource: plugin.resourceConfig, + limit: chunkSize, + offset, + filters: normalizedFilters, + sort: stableSort, + getTotals: false, + }); + + + if (data.length) { + await writer.write(buildCsvChunk(data.map((row) => serializeRow(columns, row)), columnsToForceQuote)); + exportedRows += data.length; + } + + const now = Date.now(); + if (now - lastCancellationCheckAt >= CANCELLATION_CHECK_INTERVAL_MS) { + lastCancellationCheckAt = now; + if (await isJobCancelled(plugin, jobId)) { + cancelled = true; + break; + } + } + + const isLastChunk = data.length < chunkSize; + if (!isLastChunk && now - lastProgressPublishedAt >= PROGRESS_PUBLISH_INTERVAL_MS) { + lastProgressPublishedAt = now; + await backgroundJobsPlugin.setJobStateField(jobId, 'exportedRows', exportedRows); + if (estimatedTotal > 0) { + // the estimate can be stale, so never let it reach 100% before the file is closed + await publishProgress(plugin, jobId, Math.min(99, Math.floor((exportedRows / estimatedTotal) * 100))); + } + } + + if (isLastChunk) { + break; + } + } + + if (cancelled) { + await abortQuietly(writer, fileKey); + await backgroundJobsPlugin.setJobStateField(jobId, 'exportedRows', exportedRows); + return; + } + + await writer.close(); + } catch (e) { + // leaves no dangling multipart upload behind + await abortQuietly(writer, fileKey); + throw e; + } + + await backgroundJobsPlugin.setJobStateField(jobId, 'totalRows', exportedRows); + await backgroundJobsPlugin.setJobStateField(jobId, 'exportedRows', exportedRows); + await backgroundJobsPlugin.setJobStateField(jobId, 'fileReady', true); +} + +/** + * ✅Creates the background job which exports the resource into a CSV file in the storage adapter. + */ +export async function startExport( + plugin: ImportExportPlugin, + { + filters, + sort, + adminUser, + totalRows, + }: { + filters: any; + sort: IAdminForthSort[]; + adminUser: AdminUser; + totalRows?: number; + }, +): Promise<{ ok: boolean; jobId?: string; error?: string }> { + const backgroundJobsPlugin = getBackgroundJobsPlugin(plugin); + const resourceId = plugin.resourceConfig.resourceId; + + const fileName = `export-${resourceId}-${new Date().toISOString()}.csv`; + // job id is not known before the job is created, so the timestamped name is what makes the key unique + const fileKey = `adminforth-import-export/${resourceId}/${fileName}`; + + const jobId = await backgroundJobsPlugin.startNewJob( + `Export ${resourceId} to CSV`, + adminUser, + [{ state: { filters, sort, fileKey, fileName } as TaskState }], + `${EXPORT_CSV_JOB_HANDLER_NAME}-${plugin.pluginInstanceId}`, + { + pluginInstanceId: plugin.pluginInstanceId, + resourceId, + fileName, + fileKey, + fileReady: false, + exportedRows: 0, + totalRows: totalRows ?? 0, + }, + ); + + return { ok: true, jobId }; +} + +/** + * ✅Returns a short living presigned link to the file produced by the given export job. + */ +export async function getExportDownloadUrl( + plugin: ImportExportPlugin, + jobId: string, +): Promise<{ ok: boolean; url?: string; fileName?: string; error?: string }> { + const backgroundJobsPlugin = getBackgroundJobsPlugin(plugin); + + const jobRecord = await getJobRecord(plugin, jobId); + if (!jobRecord) { + return { ok: false, error: 'Export job not found' }; + } + if (jobRecord[backgroundJobsPlugin.options.jobHandlerField] !== `${EXPORT_CSV_JOB_HANDLER_NAME}-${plugin.pluginInstanceId}`) { + return { ok: false, error: 'Export job not found' }; + } + + const state = jobRecord[backgroundJobsPlugin.options.stateField] || {}; + + if (!state.fileReady || !state.fileKey) { + return { ok: false, error: 'Export is not finished yet' }; + } + + const { storageAdapter } = plugin.options.exportBigDataset; + const url = await storageAdapter.getDownloadUrl(state.fileKey, DOWNLOAD_URL_EXPIRES_IN_SECONDS); + + return { ok: true, url, fileName: state.fileName }; +} diff --git a/index.ts b/index.ts index bd0c9d4..7abff12 100644 --- a/index.ts +++ b/index.ts @@ -1,8 +1,17 @@ import { AdminForthPlugin, suggestIfTypo, AdminForthFilterOperators, Filters, AdminForthDataTypes, rejectApiRawFilters, interpretResource, ActionCheckSource, AllowedActionsEnum } from "adminforth"; import type { IAdminForth, IHttpServer, AdminForthResourceColumn, AdminForthComponentDeclaration, AdminForthResource, AdminUser, HttpExtra, IAdminForthHttpResponse } from "adminforth"; import type { PluginOptions } from './types.js'; +import type BackgroundJobsPlugin from '@adminforth/background-jobs'; import pLimit from 'p-limit'; import { z } from "zod"; +import { + DEFAULT_READ_CHUNK_SIZE, + EXPORT_CSV_JOB_HANDLER_NAME, + MINIMAL_BUFFER_SIZE_MB, + getExportDownloadUrl, + runExportCsvJob, + startExport, +} from './exportMultipartUpload.js'; const exportCsvBodySchema = z.object({ filters: z.any(), @@ -10,6 +19,17 @@ const exportCsvBodySchema = z.object({ selectedIds: z.array(z.any()).optional(), }).strict(); +const startExportJobBodySchema = z.object({ + filters: z.any(), + sort: z.any(), + selectedIds: z.array(z.any()).optional(), + totalRows: z.number().int().nonnegative().optional(), +}).strict(); + +const exportDownloadUrlBodySchema = z.object({ + jobId: z.string(), +}).strict(); + const importCsvBodySchema = z.object({ data: z.record(z.string(), z.array(z.unknown())), }).strict(); @@ -20,7 +40,7 @@ export default class ImportExport extends AdminForthPlugin { authResourceId: string; adminforth: IAdminForth; auditLogPlugin: Record | undefined; - + backgroundJobsPlugin: any; constructor(options: PluginOptions) { super(options, import.meta.url); @@ -68,6 +88,30 @@ export default class ImportExport extends AdminForthPlugin { } } + /** + * When user exported a manual selection, the selection itself becomes the only filter. + */ + private resolveExportFilters( + filters: any, + selectedIds?: unknown[] + ): { ok: boolean; filters?: any; error?: string } { + if (!Array.isArray(selectedIds) || selectedIds.length === 0) { + return { ok: true, filters }; + } + const primaryKeyColumn = this.resourceConfig.columns.find(col => col.primaryKey); + if (!primaryKeyColumn) { + return { ok: false, error: 'Cannot export selected records: resource has no primary key' }; + } + return { + ok: true, + filters: [{ + field: primaryKeyColumn.name, + operator: AdminForthFilterOperators.IN, + value: selectedIds, + }], + }; + } + private async ensureAnyAllowed( adminUser: AdminUser, checks: { source: ActionCheckSource; action: AllowedActionsEnum }[], @@ -106,7 +150,10 @@ export default class ImportExport extends AdminForthPlugin { } (resourceConfig.options.pageInjections.list.threeDotsDropdownItems as AdminForthComponentDeclaration[]).push({ file: this.componentPath('ExportCsv.vue'), - meta: { pluginInstanceId: this.pluginInstanceId } + meta: { + pluginInstanceId: this.pluginInstanceId, + exportBigDataset: !!this.options.exportBigDataset, + } }, { file: this.componentPath('ImportCsv.vue'), meta: { pluginInstanceId: this.pluginInstanceId } @@ -123,6 +170,51 @@ export default class ImportExport extends AdminForthPlugin { } catch (e) { console.warn('Failed to get AuditLogPlugin for import-export plugin. Audit logging will be skipped.'); } + + if (this.options.exportBigDataset) { + const backgroundJobsPlugin = adminforth.getPluginByClassName('BackgroundJobsPlugin'); + + if (!backgroundJobsPlugin) { + throw new Error(`BackgroundJobsPlugin is required for export of big dataset to work, please add it to your plugins`); + } + + if (!this.options.exportBigDataset.storageAdapter) { + throw new Error(`exportBigDataset.storageAdapter is required for export of big dataset to work`); + } + + if (this.options.exportBigDataset.bufferSizeMb === undefined) { + this.options.exportBigDataset.bufferSizeMb = MINIMAL_BUFFER_SIZE_MB; + } else if (this.options.exportBigDataset.bufferSizeMb < MINIMAL_BUFFER_SIZE_MB) { + throw new Error(`exportBigDataset.bufferSizeMb must be at least ${MINIMAL_BUFFER_SIZE_MB}, got ${this.options.exportBigDataset.bufferSizeMb}`); + } + + if (this.options.exportBigDataset.readChunkSize === undefined) { + this.options.exportBigDataset.readChunkSize = DEFAULT_READ_CHUNK_SIZE; + } else if (!Number.isInteger(this.options.exportBigDataset.readChunkSize) || this.options.exportBigDataset.readChunkSize < 1) { + throw new Error(`exportBigDataset.readChunkSize must be a positive integer, got ${this.options.exportBigDataset.readChunkSize}`); + } + + this.options.exportBigDataset.storageAdapter.setupLifecycle( + `${this.resourceConfig.resourceId}-${this.pluginInstanceId}` + ); + + backgroundJobsPlugin.registerTaskHandler({ + jobHandlerName: `${EXPORT_CSV_JOB_HANDLER_NAME}-${this.pluginInstanceId}`, + handler: async ({ jobId, getState }) => { + await runExportCsvJob(this, { jobId, getState }); + }, + // whole export is a single task which streams the file, there is nothing to parallelize + parallelLimit: 1, + }) + + backgroundJobsPlugin.registerTaskDetailsComponent({ + jobHandlerName: `${EXPORT_CSV_JOB_HANDLER_NAME}-${this.pluginInstanceId}`, + component: { + file: this.componentPath('ExportCsvJobViewComponent.vue'), + meta: { pluginInstanceId: this.pluginInstanceId }, + }, + }) + } } instanceUniqueRepresentation(pluginOptions: any) : string { @@ -157,20 +249,12 @@ export default class ImportExport extends AdminForthPlugin { return rawFilterError; } - let effectiveFilters = filters; - if (Array.isArray(selectedIds) && selectedIds.length > 0) { - const primaryKeyColumn = this.resourceConfig.columns.find(col => col.primaryKey); - if (!primaryKeyColumn) { - return { ok: false, error: 'Cannot export selected records: resource has no primary key' }; - } - effectiveFilters = [{ - field: primaryKeyColumn.name, - operator: AdminForthFilterOperators.IN, - value: selectedIds, - }]; + const effectiveFilters = this.resolveExportFilters(filters, selectedIds); + if (!effectiveFilters.ok) { + return { ok: false, error: effectiveFilters.error }; } - return this.exportCsv(effectiveFilters, sort, { adminUser, headers }); + return this.exportCsv(effectiveFilters.filters, sort, { adminUser, headers }); } }); @@ -248,6 +332,91 @@ export default class ImportExport extends AdminForthPlugin { return this.checkRecords(data); } }); + + server.endpoint({ + method: 'POST', + path: `/plugin/${this.pluginInstanceId}/start-export-job`, + request_schema: startExportJobBodySchema, + handler: async ({ body, adminUser, headers }) => { + const { filters, sort, selectedIds, totalRows } = body as z.infer; + if (!this.options.exportBigDataset) { + return { ok: false, error: 'Big dataset export is not enabled for this resource' }; + } + if (!filters || !sort) { + return { ok: false, error: 'Missing filters or sort in request body' }; + } + const access = await this.ensureAnyAllowed( + adminUser, + [ + { source: ActionCheckSource.ListRequest, action: AllowedActionsEnum.list }, + { source: ActionCheckSource.ShowRequest, action: AllowedActionsEnum.show }, + ], + { requestBody: body } + ); + if (!access.ok) { + return { ok: false, error: access.error }; + } + const rawFilterError = rejectApiRawFilters(body.filters); + if (rawFilterError) { + return rawFilterError; + } + + const effectiveFilters = this.resolveExportFilters(filters, selectedIds); + if (!effectiveFilters.ok) { + return { ok: false, error: effectiveFilters.error }; + } + + const effectiveTotalRows = Array.isArray(selectedIds) && selectedIds.length > 0 + ? selectedIds.length + : totalRows; + + this.tryToAuditLogAction( + 'export', + `Started background CSV export with filters: ${JSON.stringify(effectiveFilters.filters)} and sort: ${JSON.stringify(sort)}`, + adminUser, + headers + ); + + try { + return await startExport(this, { + filters: effectiveFilters.filters, + sort, + adminUser, + totalRows: effectiveTotalRows, + }); + } catch (e) { + return { ok: false, error: e instanceof Error ? e.message : 'Failed to start export job' }; + } + }, + }) + + server.endpoint({ + method: 'POST', + path: `/plugin/${this.pluginInstanceId}/export-job-download-url`, + request_schema: exportDownloadUrlBodySchema, + handler: async ({ body, adminUser }) => { + const { jobId } = body as z.infer; + if (!this.options.exportBigDataset) { + return { ok: false, error: 'Big dataset export is not enabled for this resource' }; + } + const access = await this.ensureAnyAllowed( + adminUser, + [ + { source: ActionCheckSource.ListRequest, action: AllowedActionsEnum.list }, + { source: ActionCheckSource.ShowRequest, action: AllowedActionsEnum.show }, + ], + { requestBody: body } + ); + if (!access.ok) { + return { ok: false, error: access.error }; + } + try { + return await getExportDownloadUrl(this, jobId); + } catch (e) { + return { ok: false, error: e instanceof Error ? e.message : 'Failed to build download link' }; + } + }, + }) } /** diff --git a/package.json b/package.json index 4aabf7e..c31b051 100644 --- a/package.json +++ b/package.json @@ -28,8 +28,9 @@ "adminforth": "^3.8.2" }, "devDependencies": { + "@adminforth/background-jobs": "^1.18.4", "@types/node": "^22.10.7", - "adminforth": "^3.8.2", + "adminforth": "^3.12.1", "semantic-release": "^24.2.1", "semantic-release-slack-bot": "^4.0.2", "typescript": "^5.7.3" @@ -60,6 +61,7 @@ ] }, "dependencies": { + "csv": "^6.6.1", "p-limit": "^7.3.0", "zod": "^4.3.6" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0e53900..2ae0f69 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: dependencies: + csv: + specifier: ^6.6.1 + version: 6.6.1 p-limit: specifier: ^7.3.0 version: 7.3.0 @@ -15,12 +18,15 @@ importers: specifier: ^4.3.6 version: 4.3.6 devDependencies: + '@adminforth/background-jobs': + specifier: ^1.18.4 + version: 1.18.4(adminforth@3.12.1)(vue@3.5.40(typescript@5.9.3)) '@types/node': specifier: ^22.10.7 version: 22.19.13 adminforth: - specifier: ^3.8.2 - version: 3.8.2(@adminforth/connector-qdrant@1.0.4)(@types/node@22.19.13) + specifier: ^3.12.1 + version: 3.12.1(@adminforth/connector-qdrant@1.0.4)(@types/node@22.19.13)(express@4.22.1) semantic-release: specifier: ^24.2.1 version: 24.2.9(typescript@5.9.3) @@ -33,6 +39,11 @@ importers: packages: + '@adminforth/background-jobs@1.18.4': + resolution: {integrity: sha512-znE3hJYjGeD4yUh4Aefh0TWU/HzfWPYybYCjGzpAjt3HRrGdd1XBLVw6TOCPsjgmwEi6f8IJkiuNo3swYaVjDA==} + peerDependencies: + adminforth: ^3.8.2 + '@adminforth/connector-qdrant@1.0.4': resolution: {integrity: sha512-ThblJRLSKLPHBOXNeJ8wCj+3sxOgFa0bEj1JX3s3Kl+XpZmAHChECgXLErS+IyqaAobjjmWck9jIO2dP917gGw==} peerDependencies: @@ -46,19 +57,36 @@ packages: resolution: {integrity: sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==} engines: {node: '>=6.9.0'} + '@babel/helper-string-parser@7.29.7': + resolution: {integrity: sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==} + engines: {node: '>=6.9.0'} + '@babel/helper-validator-identifier@7.28.5': resolution: {integrity: sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==} engines: {node: '>=6.9.0'} + '@babel/helper-validator-identifier@7.29.7': + resolution: {integrity: sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==} + engines: {node: '>=6.9.0'} + '@babel/parser@7.29.0': resolution: {integrity: sha512-IyDgFV5GeDUVX4YdF/3CPULtVGSXXMLh1xVIgdCgxApktqnQV0r7/8Nqthg+8YLGaAtdyIlo2qIdZrbCv4+7ww==} engines: {node: '>=6.0.0'} hasBin: true + '@babel/parser@7.29.7': + resolution: {integrity: sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg==} + engines: {node: '>=6.0.0'} + hasBin: true + '@babel/types@7.29.0': resolution: {integrity: sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==} engines: {node: '>=6.9.0'} + '@babel/types@7.29.7': + resolution: {integrity: sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==} + engines: {node: '>=6.9.0'} + '@chainsafe/is-ip@2.1.0': resolution: {integrity: sha512-KIjt+6IfysQ4GCv66xihEitBjvhU/bixbbbFxdJ1sqCp4uJ0wuZiYBPhksZoy4lfaF0k9cwNzY5upEW/VWdw3w==} @@ -204,6 +232,9 @@ packages: '@types/node': optional: true + '@jridgewell/sourcemap-codec@1.5.5': + resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} + '@octokit/auth-token@6.0.0': resolution: {integrity: sha512-P4YJBPdPSpWTQ1NU4XYdvHvXJJDxM6YwpS0FZHRgP7YFkdVxsWcpWGy/NVqlAA7PcPCnMacXlRm1y2PFZRWL/w==} engines: {node: '>= 20'} @@ -351,11 +382,11 @@ packages: '@types/connect@3.4.38': resolution: {integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==} - '@types/express-serve-static-core@4.19.8': - resolution: {integrity: sha512-02S5fmqeoKzVZCHPZid4b8JH2eM5HzQLZWN2FohQEy/0eXTq8VXZfSN6Pcr3F6N9R/vNrj7cpgbhjie6m/1tCA==} + '@types/express-serve-static-core@5.1.2': + resolution: {integrity: sha512-d3KvEXBSo/lOAMc2u6fkyDHBvetBHeqD7wm/AcXfLpSOQwlmG9D/aQ0SFswVjv05p7ullQS7Mjohj6/VdbZuTg==} - '@types/express@4.17.25': - resolution: {integrity: sha512-dVd04UKsfpINUnK0yBoYHDF3xu7xVH4BuDotC/xGuycx4CgbP48X/KF/586bcObxT0HENHXEU8Nqtu6NR+eKhw==} + '@types/express@5.0.6': + resolution: {integrity: sha512-sKYVuV7Sv9fbPIt/442koC7+IIwK5olP1KWeD88e/idgoJqDm3JV/YUiPwkoKK92ylff2MGxSz1CSjsXelx0YA==} '@types/http-errors@2.0.5': resolution: {integrity: sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==} @@ -363,9 +394,6 @@ packages: '@types/mdast@3.0.15': resolution: {integrity: sha512-LnwD+mUEfxWMa1QpDraczIn6k0Ee3SMicuYSSzS6ZYl2gKS09EClnJYGd8Du6rfc5r/GZEk5o1mRb8TaTj03sQ==} - '@types/mime@1.3.5': - resolution: {integrity: sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==} - '@types/node@22.19.13': resolution: {integrity: sha512-akNQMv0wW5uyRpD2v2IEyRSZiR+BeGuoB6L310EgGObO44HSMNT8z1xzio28V8qOrgYaopIDNA18YgdXd+qTiw==} @@ -378,24 +406,68 @@ packages: '@types/range-parser@1.2.7': resolution: {integrity: sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==} - '@types/send@0.17.6': - resolution: {integrity: sha512-Uqt8rPBE8SY0RK8JB1EzVOIZ32uqy8HwdxCnoCOsYrvnswqmFZ/k+9Ikidlk/ImhsdvBsloHbAlewb2IEBV/Og==} - '@types/send@1.2.1': resolution: {integrity: sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ==} - '@types/serve-static@1.15.10': - resolution: {integrity: sha512-tRs1dB+g8Itk72rlSI2ZrW6vZg0YrLI81iQSTkMmOqnqCaNr/8Ek4VwWcN5vZgCYWbg/JJSGBlUaYGAOP73qBw==} + '@types/serve-static@2.2.0': + resolution: {integrity: sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ==} '@types/unist@2.0.11': resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==} + '@types/web-bluetooth@0.0.21': + resolution: {integrity: sha512-oIQLCGWtcFZy2JW77j9k8nHzAOpqMHLQejDA48XXMWH6tjCQHz5RCFz1bzsmROyL6PUm+LLnUiI4BCn221inxA==} + + '@vue/compiler-core@3.5.40': + resolution: {integrity: sha512-39E8IgOhTbVDnoJFMKc2DvYnypcZwUqgUhQkccva/0m6FUwtIKSGV7n1hpVmYcFaoRAwf9pBcwnKlCEsN63ZEQ==} + + '@vue/compiler-dom@3.5.40': + resolution: {integrity: sha512-pwkx4vqlqOspFstrcmzwkKLePVMD3PT65imRzLhanU2V1Fj4K13g6OXjanOyzw3aTAuRk84BOmY8f3rEHqPaVA==} + + '@vue/compiler-sfc@3.5.40': + resolution: {integrity: sha512-gIf497P4kpuALcvs5n3AEg1Vdn0pSY4XbjASIfHNYF1/MP3T2Mf2STERTubysBxCRxzJGJYtF/O7vwJrxFB3Vw==} + + '@vue/compiler-ssr@3.5.40': + resolution: {integrity: sha512-rrE5xiXG663+vHCHa3J9p2z5OcBRjXmoqenprJxAFQxg5pSshzeBiCE6pu46axapRJ2Adk0YDA2BRZVjiHXnhg==} + + '@vue/reactivity@3.5.40': + resolution: {integrity: sha512-B7ot9UlUZOi1zbq61/LvE88ZLTV8IlajTdiZTAEiDQgrnIMIZoPr9kGw0Zw46ObW62O9+H/Be3kMbfb7kYPQZA==} + + '@vue/runtime-core@3.5.40': + resolution: {integrity: sha512-KAZLweuZ6uUJPK1PMSQPgBU5gCjgrrfjUhSglmU9NhH+Zjepa8cnwSydPWDWHDwOgY4g3VcZ+PljbiHlURNCbw==} + + '@vue/runtime-dom@3.5.40': + resolution: {integrity: sha512-ZfrX8ssZQds900L9pr8AuK05ddnMsR4MPMZr8cPN9GoqoPWcXLhjvvbIA2SMv+7a97sJ1vv9pj/zxK0Cq/eEFQ==} + + '@vue/server-renderer@3.5.40': + resolution: {integrity: sha512-XNJym9WpevhTVt1HuwOrCRJ5Q+9z4BjTMrDtjTrvx74SmUll8spNTw6whWJa9mEkO4PKn5TihI/bm/8ds2QVJw==} + + '@vue/shared@3.5.40': + resolution: {integrity: sha512-WxnBtruIqOoV3rA4jeKDWzrYI5h7Cp4+pjwDi8kWGHz+IslhiN+wguLVVhtv2l8VoU02rzDCVfDjgCl1lNpZVg==} + + '@vueuse/core@14.3.0': + resolution: {integrity: sha512-aHfz47g0ZhMtTVHmIzMVpJy8ePhhOy68GY5bv110+5DVtZ+W7BsOx+m61UNQqfrWyPztIHIanWa3E2tib3NFIw==} + peerDependencies: + vue: ^3.5.0 + + '@vueuse/metadata@14.3.0': + resolution: {integrity: sha512-BwxmbAzwAVF50+MW57GXOUEV61nFBGnlBvrTqj49PqWJu3uw7hdu72ztXeZ33RdZtDY6kO+bfCAE1PCn88Tktw==} + + '@vueuse/shared@14.3.0': + resolution: {integrity: sha512-bZpge9eSXwa4ToSiqJ7j6KRwhAsneMFoSz3LMWKQDkqimm3D/tbFlrklrs/IOqC8tEcYmXQZJ6N0UrjhBirVCg==} + peerDependencies: + vue: ^3.5.0 + + abstract-level@3.1.1: + resolution: {integrity: sha512-CW2gKbJFTuX1feMvOrvsVMmijAOgI9kg2Ie9Dq3gOcMt/dVVoVmqNlLcEUCT13NxHFMEajcUcVBIplbyDroDiw==} + engines: {node: '>=18'} + accepts@1.3.8: resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} engines: {node: '>= 0.6'} - adminforth@3.8.2: - resolution: {integrity: sha512-d52t5LQP22EFWugZJ8xqPARX+EF1G3ieI0yfM0UPmWDLe1ZwkU+I1BUYyaRBcCWlaqf48EsFPkg6GJzRIpefMA==} + adminforth@3.12.1: + resolution: {integrity: sha512-nP2N/6FIAoXTbHcwetKzjAjzu52G1Zv7pWzjOH6rsqh0eVhDJfBFByKfdTbjNlNI6xmq1dzRzTlEilnQbk5SMA==} hasBin: true peerDependencies: '@adminforth/connector-clickhouse': ^1.0.4 @@ -404,6 +476,7 @@ packages: '@adminforth/connector-postgres': ^1.0.4 '@adminforth/connector-qdrant': ^1.0.4 '@adminforth/connector-sqlite': ^1.0.3 + express: ^4.21.0 || ^5.0.0 peerDependenciesMeta: '@adminforth/connector-clickhouse': optional: true @@ -482,6 +555,9 @@ packages: resolution: {integrity: sha512-6t10qk83GOG8p0vKmaCr8eiilZwO171AvbROMtvvNiwrTly62t+7XkA8RdIIVbpMhCASAsxgAzdRSwh6nw/5Dg==} engines: {node: '>=4'} + async-mutex@0.5.0: + resolution: {integrity: sha512-1A94B18jkJ3DYq284ohPxoXbfTA5HsQ7/Mf4DEhcyLx3Bz27Rh59iScbB6EPiP+B+joue6YCxcMXSbFC1tZKwA==} + atomic-sleep@1.0.0: resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==} engines: {node: '>=8.0.0'} @@ -489,6 +565,9 @@ packages: bail@1.0.5: resolution: {integrity: sha512-xFbRxM1tahm08yHBP16MMjVUAvDaBMD38zsM9EMAUN61omwLmKlOpB/Zku5QkjZ8TZ4vn53pj+t518cH0S03RQ==} + base64-js@1.5.1: + resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + before-after-hook@4.0.0: resolution: {integrity: sha512-q6tR3RPqIB1pMiTRMFcZwuG5T8vwp+vUvEG0vuI6B+Rikh5BfPp2fQ82c925FOs+b0lcFQ8CFrL+KbilfZFhOQ==} @@ -503,6 +582,9 @@ packages: resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} engines: {node: '>=8'} + browser-level@3.0.0: + resolution: {integrity: sha512-kGXtLh29jMwqKaskz5xeDLtCtN1KBz/DbQSqmvH7QdJiyGRC7RAM8PPg6gvUiNMa+wVnaxS9eSmEtP/f5ajOVw==} + bson@7.2.0: resolution: {integrity: sha512-YCEo7KjMlbNlyHhz7zAZNDpIpQbd+wOEHJYezv0nMYTn4x31eIUM2yomNNubclAt63dObUzKHWsBLJ9QcZNSnQ==} engines: {node: '>=20.19.0'} @@ -513,6 +595,9 @@ packages: buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} + buffer@6.0.3: + resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} + busboy@1.6.0: resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==} engines: {node: '>=10.16.0'} @@ -564,6 +649,10 @@ packages: chardet@2.1.1: resolution: {integrity: sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ==} + classic-level@3.0.0: + resolution: {integrity: sha512-yGy8j8LjPbN0Bh3+ygmyYvrmskVita92pD/zCoalfcC9XxZj6iDtZTAnz+ot7GG8p9KLTG+MZ84tSA4AhkgVZQ==} + engines: {node: '>=18'} + clean-stack@5.3.0: resolution: {integrity: sha512-9ngPTOhYGQqNVSfeJkYXHmF7AGWp4/nN5D/QqNQs3Dvxd1Kk/WpjHfNujKHYUQ/5CoGyOyFNoWSPk5afzP0QVg==} engines: {node: '>=14.16'} @@ -683,6 +772,22 @@ packages: resolution: {integrity: sha512-x8dy3RnvYdlUcPOjkEHqozhiwzKNSq7GcPuXFbnyMOCHxX8V3OgIg/pYuabl2sbUPfIJaeAQB7PMOK8DFIdoRA==} engines: {node: '>=12'} + csstype@3.2.3: + resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} + + csv-generate@4.6.1: + resolution: {integrity: sha512-eELl9K716LSSeP2/YcCjch525JztnnERe3jEARWw2v1FN9ukUYfZTNYZ4Rq2Jj/MFKMauffOy9VCaqQTpFThDQ==} + + csv-parse@7.0.1: + resolution: {integrity: sha512-+2z7Ar0APQ7Uu6fX4cn+pitRmxjZ1WPBcGmZFKmA74FCyi7Et/XZx8cjNQ5CjbZ4HCOxXCOpRBYvYH08Qa003A==} + + csv-stringify@6.8.1: + resolution: {integrity: sha512-tZ6X6TKQyQgCo5OptXcyAbfN1pwmoxEqELPQ7KFazNErx7kiVsDK8o+VYRXhfMl4N9vvOOLXuioquR2MeP847A==} + + csv@6.6.1: + resolution: {integrity: sha512-QKTUoSvOaJQGL23LxlGqQiy1n0g2Eg/T72tzsKqtX7APrchpwBpf0SJMX2fzrrXj87wkDAPQBgvoSAVKqKAYHA==} + engines: {node: '>= 0.1.90'} + data-uri-to-buffer@4.0.1: resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} engines: {node: '>= 12'} @@ -846,6 +951,9 @@ packages: engines: {node: '>=4'} hasBin: true + estree-walker@2.0.2: + resolution: {integrity: sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==} + etag@1.8.1: resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} engines: {node: '>= 0.6'} @@ -1072,6 +1180,9 @@ packages: resolution: {integrity: sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==} engines: {node: '>=0.10.0'} + ieee754@1.2.1: + resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} + import-fresh@3.3.1: resolution: {integrity: sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==} engines: {node: '>=6'} @@ -1242,6 +1353,18 @@ packages: launder@1.7.1: resolution: {integrity: sha512-mU6WRz5EusL9ZZuiZ5SO4Y6C0P9PAUR9iwdb6bzj4KDihm28DiHFw+/yk9DBH4f+Pv1wuzQ4e2jV3oQ7mkIqvw==} + level-supports@6.2.0: + resolution: {integrity: sha512-QNxVXP0IRnBmMsJIh+sb2kwNCYcKciQZJEt+L1hPCHrKNELllXhvrlClVHXBYZVT+a7aTSM6StgNXdAldoab3w==} + engines: {node: '>=16'} + + level-transcoder@1.0.1: + resolution: {integrity: sha512-t7bFwFtsQeD8cl8NIoQ2iwxA0CL/9IFw7/9gAjOonH0PWTTiRfY7Hq+Ejbsxh86tXobDQ6IOiddjNYIfOBs06w==} + engines: {node: '>=12'} + + level@10.0.0: + resolution: {integrity: sha512-aZJvdfRr/f0VBbSRF5C81FHON47ZsC2TkGxbBezXpGGXAUEL/s6+GP73nnhAYRSCIqUNsmJjfeOF4lzRDKbUig==} + engines: {node: '>=18'} + lines-and-columns@1.2.4: resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} @@ -1300,6 +1423,9 @@ packages: lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} + magic-string@0.30.21: + resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + make-asynchronous@1.1.0: resolution: {integrity: sha512-ayF7iT+44LXdxJLTrTd3TLQpFDDvPCBxXxbv+pMUSuHA5Q8zyAfwkRP6aHHwNVFBUFWtxAHqwNJxF8vMZLAbVg==} engines: {node: '>=18'} @@ -1322,6 +1448,10 @@ packages: resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} engines: {node: '>= 0.4'} + maybe-combine-errors@1.0.0: + resolution: {integrity: sha512-eefp6IduNPT6fVdwPp+1NgD0PML1NU5P6j1Mj5nz1nidX8/sWY7119WL8vTAHgqfsY74TzW0w1XPgdYEKkGZ5A==} + engines: {node: '>=10'} + mdast-util-find-and-replace@1.1.1: resolution: {integrity: sha512-9cKl33Y21lyckGzpSmEQnIDjEfeeWelN5s1kUW1LwdB0Fkuq2u+4GdqcGEygYxJE8GVqCl0741bYXHgamfWAZA==} @@ -1425,6 +1555,10 @@ packages: minimist@1.2.8: resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + module-error@1.0.2: + resolution: {integrity: sha512-0yuvsqSCv8LbaOKhnsQ/T5JhyFlCYLPXK3U2sgV10zoKQwzs/MyfuQUOZQ1V/6OCOJsK/TRgNVrPuPDqtdMFtA==} + engines: {node: '>=10'} + ms@2.0.0: resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} @@ -1447,11 +1581,19 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true + nanoid@3.3.16: + resolution: {integrity: sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + nanoid@5.1.9: resolution: {integrity: sha512-ZUvP7KeBLe3OZ1ypw6dI/TzYJuvHP77IM4Ry73waSQTLn8/g8rpdjfyVAh7t1/+FjBtG4lCP42MEbDxOsRpBMw==} engines: {node: ^18 || >=20} hasBin: true + napi-macros@2.2.2: + resolution: {integrity: sha512-hmEVtAGYzVQpCKdbQea4skABsdXW4RUh5t5mJ2zzqowJS2OyXZTU1KhDVFhx+NlWZ4ap9mqR9TcDO3LTTttd+g==} + negotiator@0.6.3: resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} engines: {node: '>= 0.6'} @@ -1488,6 +1630,10 @@ packages: resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + node-gyp-build@4.8.4: + resolution: {integrity: sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==} + hasBin: true + normalize-package-data@6.0.2: resolution: {integrity: sha512-V6gygoYb/5EmNI+MEGrWkC+e6+Rr7mTmfHrxDbLzxQogBkgzo76rkok0Am6thgSF7Mv2nLOajAJj5vDJZEFn7g==} engines: {node: ^16.14.0 || >=18.0.0} @@ -1744,6 +1890,10 @@ packages: resolution: {integrity: sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==} engines: {node: ^10 || ^12 || >=14} + postcss@8.5.23: + resolution: {integrity: sha512-g50586zr4bZmwFiTlflMu8E0bDTb5I5gertgwAKmsdUlTQIhZtunzUlD1WSzwcVWPoAVpsrA6vlfCD7oXvRwgg==} + engines: {node: ^10 || ^12 || >=14} + pretty-ms@9.3.0: resolution: {integrity: sha512-gjVS5hOP+M3wMm5nmNOucbIrqudzs9v/57bWRHQWLYklXqoXKrVfYW2W9+glfGsqtPgpiz5WwyEEB+ksXIx3gQ==} engines: {node: '>=18'} @@ -2002,6 +2152,9 @@ packages: stream-combiner2@1.1.1: resolution: {integrity: sha512-3PnJbYgS56AeWgtKF5jtJRT6uFJe56Z0Hc5Ngg/6sI6rIt8iiMBTa9cvdyFfpMQjaVHr8dusbNeFGIIonxOvKw==} + stream-transform@3.5.1: + resolution: {integrity: sha512-TTDX+qKFr7GGRXATn66rprmlFPx08W0UBIccE/rMPgW2sT7GovduZYP4xcdJ7Nu2YigF17U+CNFxYY11+W1oPw==} + streamsearch@1.1.0: resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} engines: {node: '>=10.0.0'} @@ -2231,6 +2384,14 @@ packages: vfile@4.2.1: resolution: {integrity: sha512-O6AE4OskCG5S1emQ/4gl8zK586RqA3srz3nfK/Viy0UPToBc5Trp9BVFb1u0CjsKrAWwnpr4ifM/KBXPWwJbCA==} + vue@3.5.40: + resolution: {integrity: sha512-+8PJ4SJXdn/cHGImF4CKdxlWHIN5Dkt7DoufRREM6h6uVCx2m7QxgcEQmmzyOK8A9mcafg7sFbJFYsdFVubTig==} + peerDependencies: + typescript: '*' + peerDependenciesMeta: + typescript: + optional: true + web-streams-polyfill@3.3.3: resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} engines: {node: '>= 8'} @@ -2328,10 +2489,21 @@ packages: snapshots: - '@adminforth/connector-qdrant@1.0.4(adminforth@3.8.2)(typescript@5.9.3)': + '@adminforth/background-jobs@1.18.4(adminforth@3.12.1)(vue@3.5.40(typescript@5.9.3))': + dependencies: + '@vueuse/core': 14.3.0(vue@3.5.40(typescript@5.9.3)) + adminforth: 3.12.1(@adminforth/connector-qdrant@1.0.4)(@types/node@22.19.13)(express@4.22.1) + async-mutex: 0.5.0 + level: 10.0.0 + p-limit: 7.3.0 + zod: 4.3.6 + transitivePeerDependencies: + - vue + + '@adminforth/connector-qdrant@1.0.4(adminforth@3.12.1)(typescript@5.9.3)': dependencies: '@qdrant/js-client-rest': 1.18.0(typescript@5.9.3) - adminforth: 3.8.2(@adminforth/connector-qdrant@1.0.4)(@types/node@22.19.13) + adminforth: 3.12.1(@adminforth/connector-qdrant@1.0.4)(@types/node@22.19.13)(express@4.22.1) dayjs: 1.11.21 transitivePeerDependencies: - typescript @@ -2345,17 +2517,30 @@ snapshots: '@babel/helper-string-parser@7.27.1': {} + '@babel/helper-string-parser@7.29.7': {} + '@babel/helper-validator-identifier@7.28.5': {} + '@babel/helper-validator-identifier@7.29.7': {} + '@babel/parser@7.29.0': dependencies: '@babel/types': 7.29.0 + '@babel/parser@7.29.7': + dependencies: + '@babel/types': 7.29.7 + '@babel/types@7.29.0': dependencies: '@babel/helper-string-parser': 7.27.1 '@babel/helper-validator-identifier': 7.28.5 + '@babel/types@7.29.7': + dependencies: + '@babel/helper-string-parser': 7.29.7 + '@babel/helper-validator-identifier': 7.29.7 + '@chainsafe/is-ip@2.1.0': {} '@colors/colors@1.5.0': @@ -2488,6 +2673,8 @@ snapshots: optionalDependencies: '@types/node': 22.19.13 + '@jridgewell/sourcemap-codec@1.5.5': {} + '@octokit/auth-token@6.0.0': {} '@octokit/core@7.0.6': @@ -2685,19 +2872,18 @@ snapshots: dependencies: '@types/node': 22.19.13 - '@types/express-serve-static-core@4.19.8': + '@types/express-serve-static-core@5.1.2': dependencies: '@types/node': 22.19.13 '@types/qs': 6.14.0 '@types/range-parser': 1.2.7 '@types/send': 1.2.1 - '@types/express@4.17.25': + '@types/express@5.0.6': dependencies: '@types/body-parser': 1.19.6 - '@types/express-serve-static-core': 4.19.8 - '@types/qs': 6.14.0 - '@types/serve-static': 1.15.10 + '@types/express-serve-static-core': 5.1.2 + '@types/serve-static': 2.2.0 '@types/http-errors@2.0.5': {} @@ -2705,8 +2891,6 @@ snapshots: dependencies: '@types/unist': 2.0.11 - '@types/mime@1.3.5': {} - '@types/node@22.19.13': dependencies: undici-types: 6.21.0 @@ -2717,35 +2901,107 @@ snapshots: '@types/range-parser@1.2.7': {} - '@types/send@0.17.6': - dependencies: - '@types/mime': 1.3.5 - '@types/node': 22.19.13 - '@types/send@1.2.1': dependencies: '@types/node': 22.19.13 - '@types/serve-static@1.15.10': + '@types/serve-static@2.2.0': dependencies: '@types/http-errors': 2.0.5 '@types/node': 22.19.13 - '@types/send': 0.17.6 '@types/unist@2.0.11': {} + '@types/web-bluetooth@0.0.21': {} + + '@vue/compiler-core@3.5.40': + dependencies: + '@babel/parser': 7.29.7 + '@vue/shared': 3.5.40 + entities: 7.0.1 + estree-walker: 2.0.2 + source-map-js: 1.2.1 + + '@vue/compiler-dom@3.5.40': + dependencies: + '@vue/compiler-core': 3.5.40 + '@vue/shared': 3.5.40 + + '@vue/compiler-sfc@3.5.40': + dependencies: + '@babel/parser': 7.29.7 + '@vue/compiler-core': 3.5.40 + '@vue/compiler-dom': 3.5.40 + '@vue/compiler-ssr': 3.5.40 + '@vue/shared': 3.5.40 + estree-walker: 2.0.2 + magic-string: 0.30.21 + postcss: 8.5.23 + source-map-js: 1.2.1 + + '@vue/compiler-ssr@3.5.40': + dependencies: + '@vue/compiler-dom': 3.5.40 + '@vue/shared': 3.5.40 + + '@vue/reactivity@3.5.40': + dependencies: + '@vue/shared': 3.5.40 + + '@vue/runtime-core@3.5.40': + dependencies: + '@vue/reactivity': 3.5.40 + '@vue/shared': 3.5.40 + + '@vue/runtime-dom@3.5.40': + dependencies: + '@vue/reactivity': 3.5.40 + '@vue/runtime-core': 3.5.40 + '@vue/shared': 3.5.40 + csstype: 3.2.3 + + '@vue/server-renderer@3.5.40': + dependencies: + '@vue/compiler-ssr': 3.5.40 + '@vue/runtime-dom': 3.5.40 + '@vue/shared': 3.5.40 + + '@vue/shared@3.5.40': {} + + '@vueuse/core@14.3.0(vue@3.5.40(typescript@5.9.3))': + dependencies: + '@types/web-bluetooth': 0.0.21 + '@vueuse/metadata': 14.3.0 + '@vueuse/shared': 14.3.0(vue@3.5.40(typescript@5.9.3)) + vue: 3.5.40(typescript@5.9.3) + + '@vueuse/metadata@14.3.0': {} + + '@vueuse/shared@14.3.0(vue@3.5.40(typescript@5.9.3))': + dependencies: + vue: 3.5.40(typescript@5.9.3) + + abstract-level@3.1.1: + dependencies: + buffer: 6.0.3 + is-buffer: 2.0.5 + level-supports: 6.2.0 + level-transcoder: 1.0.1 + maybe-combine-errors: 1.0.0 + module-error: 1.0.2 + accepts@1.3.8: dependencies: mime-types: 2.1.35 negotiator: 0.6.3 - adminforth@3.8.2(@adminforth/connector-qdrant@1.0.4)(@types/node@22.19.13): + adminforth@3.12.1(@adminforth/connector-qdrant@1.0.4)(@types/node@22.19.13)(express@4.22.1): dependencies: '@babel/parser': 7.29.0 '@faker-js/faker': 9.9.0 '@inquirer/prompts': 7.10.1(@types/node@22.19.13) '@scalar/express-api-reference': 0.9.9 - '@types/express': 4.17.25 + '@types/express': 5.0.6 ajv: 8.18.0 arg: 5.0.2 ast-types: 0.14.2 @@ -2777,11 +3033,10 @@ snapshots: yaml: 2.8.3 zod: 4.3.6 optionalDependencies: - '@adminforth/connector-qdrant': 1.0.4(adminforth@3.8.2)(typescript@5.9.3) + '@adminforth/connector-qdrant': 1.0.4(adminforth@3.12.1)(typescript@5.9.3) transitivePeerDependencies: - '@types/node' - bufferutil - - supports-color - utf-8-validate agent-base@7.1.4: {} @@ -2838,10 +3093,16 @@ snapshots: dependencies: tslib: 2.8.1 + async-mutex@0.5.0: + dependencies: + tslib: 2.8.1 + atomic-sleep@1.0.0: {} bail@1.0.5: {} + base64-js@1.5.1: {} + before-after-hook@4.0.0: {} body-parser@1.20.4: @@ -2867,12 +3128,21 @@ snapshots: dependencies: fill-range: 7.1.1 + browser-level@3.0.0: + dependencies: + abstract-level: 3.1.1 + bson@7.2.0: {} buffer-equal-constant-time@1.0.1: {} buffer-from@1.1.2: {} + buffer@6.0.3: + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + busboy@1.6.0: dependencies: streamsearch: 1.1.0 @@ -2916,6 +3186,13 @@ snapshots: chardet@2.1.1: {} + classic-level@3.0.0: + dependencies: + abstract-level: 3.1.1 + module-error: 1.0.2 + napi-macros: 2.2.2 + node-gyp-build: 4.8.4 + clean-stack@5.3.0: dependencies: escape-string-regexp: 5.0.0 @@ -3043,6 +3320,21 @@ snapshots: dependencies: type-fest: 1.4.0 + csstype@3.2.3: {} + + csv-generate@4.6.1: {} + + csv-parse@7.0.1: {} + + csv-stringify@6.8.1: {} + + csv@6.6.1: + dependencies: + csv-generate: 4.6.1 + csv-parse: 7.0.1 + csv-stringify: 6.8.1 + stream-transform: 3.5.1 + data-uri-to-buffer@4.0.1: {} dateformat@4.6.3: {} @@ -3162,6 +3454,8 @@ snapshots: esprima@4.0.1: {} + estree-walker@2.0.2: {} + etag@1.8.1: {} eventemitter3@5.0.4: {} @@ -3436,6 +3730,8 @@ snapshots: dependencies: safer-buffer: 2.1.2 + ieee754@1.2.1: {} + import-fresh@3.3.1: dependencies: parent-module: 1.0.1 @@ -3586,6 +3882,19 @@ snapshots: dependencies: dayjs: 1.11.21 + level-supports@6.2.0: {} + + level-transcoder@1.0.1: + dependencies: + buffer: 6.0.3 + module-error: 1.0.2 + + level@10.0.0: + dependencies: + abstract-level: 3.1.1 + browser-level: 3.0.0 + classic-level: 3.0.0 + lines-and-columns@1.2.4: {} listr2@8.3.3: @@ -3643,6 +3952,10 @@ snapshots: lru-cache@10.4.3: {} + magic-string@0.30.21: + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + make-asynchronous@1.1.0: dependencies: p-event: 6.0.1 @@ -3668,6 +3981,8 @@ snapshots: math-intrinsics@1.1.0: {} + maybe-combine-errors@1.0.0: {} + mdast-util-find-and-replace@1.1.1: dependencies: escape-string-regexp: 4.0.0 @@ -3806,6 +4121,8 @@ snapshots: minimist@1.2.8: {} + module-error@1.0.2: {} + ms@2.0.0: {} ms@2.1.3: {} @@ -3827,8 +4144,12 @@ snapshots: nanoid@3.3.12: {} + nanoid@3.3.16: {} + nanoid@5.1.9: {} + napi-macros@2.2.2: {} + negotiator@0.6.3: {} neo-async@2.6.2: {} @@ -3856,6 +4177,8 @@ snapshots: fetch-blob: 3.2.0 formdata-polyfill: 4.0.10 + node-gyp-build@4.8.4: {} + normalize-package-data@6.0.2: dependencies: hosted-git-info: 7.0.2 @@ -4039,6 +4362,12 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 + postcss@8.5.23: + dependencies: + nanoid: 3.3.16 + picocolors: 1.1.1 + source-map-js: 1.2.1 + pretty-ms@9.3.0: dependencies: parse-ms: 4.0.0 @@ -4382,6 +4711,8 @@ snapshots: duplexer2: 0.1.4 readable-stream: 2.3.8 + stream-transform@3.5.1: {} + streamsearch@1.1.0: {} string-width@4.2.3: @@ -4593,6 +4924,16 @@ snapshots: unist-util-stringify-position: 2.0.3 vfile-message: 2.0.4 + vue@3.5.40(typescript@5.9.3): + dependencies: + '@vue/compiler-dom': 3.5.40 + '@vue/compiler-sfc': 3.5.40 + '@vue/runtime-dom': 3.5.40 + '@vue/server-renderer': 3.5.40 + '@vue/shared': 3.5.40 + optionalDependencies: + typescript: 5.9.3 + web-streams-polyfill@3.3.3: {} web-worker@1.5.0: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index a454377..1c29266 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,4 +1,5 @@ allowBuilds: adminforth: true + classic-level: true minimumReleaseAgeExclude: - adminforth diff --git a/types.ts b/types.ts index 0ae7a63..4440001 100644 --- a/types.ts +++ b/types.ts @@ -1,5 +1,24 @@ -import {type PluginsCommonOptions } from "adminforth"; +import {type PluginsCommonOptions, StorageAdapter } from "adminforth"; export interface PluginOptions extends PluginsCommonOptions { + /** + * If you are going to export a huge dataset - it can make your server run out of memory, because all exported records are stored in RAM + * To prevent this, you can use exportBigDataset option. + * It requires a background jobs plugin to be setted up + */ + exportBigDataset?: { + /** + * Size of the buffer, where records will be stored before upload. Min size is 5 MB. The default value is 5 MB. (takes RAM memory on the server) + * AWS S3 doesn't support more that 10000 parts per upload, so if you have a huge database you want to export (for default settings more that 100 Gb) - increase buffer size + */ + bufferSizeMb?: number; + /** + * How many records are pulled from the database per iteration. The default value is 100. + * Bigger values make the export faster but take more RAM on the server, smaller values do the opposite. + */ + readChunkSize?: number; + + storageAdapter: StorageAdapter; + } } \ No newline at end of file