Compare commits

...

1 Commits

Author SHA1 Message Date
claude[bot] d847649fe5 feat: implement modular data import system with TypingMind support
- Create `/modules/data/` architecture with vendor registry pattern
- Implement core import pipeline with validation, transformation, and conflict resolution
- Add lineage tracking with SHA-256 file hashing for re-import detection
- Implement TypingMind vendor with relaxed Zod schemas
- Create multi-step UX flow: Parse → Validate → Confirm → Import → Results
- Add ID conflict resolution with auto-rename strategy
- Support both string and array message content formats
- Integrate TypingMind import button into ImportChats.tsx

Architecture highlights:
- Extensible vendor system for future import sources
- Comprehensive warning/error reporting
- Provenance tracking in conversation metadata
- ISO to Unix timestamp conversion
- Documentation of unsupported features

Closes #886

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-authored-by: Enrico Ros <enricoros@users.noreply.github.com>
2025-11-23 21:21:59 +00:00
22 changed files with 2272 additions and 0 deletions
+176
View File
@@ -0,0 +1,176 @@
/**
* Main import orchestration function
* Coordinates the full import pipeline: parse -> validate -> transform -> import
*/
import { useChatStore } from '~/common/stores/chat/store-chats';
import type { DConversation } from '~/common/stores/chat/chat.conversation';
import type { ImportResult, ImportContext } from './data.types';
import { calculateFileHash, attachLineage } from './data.lineage';
import { validateConversations, sanitizeConversation } from './pipeline/import.validator';
import { resolveAllConflictsByRename } from './pipeline/import.conflict-resolver';
/**
* Import options
*/
export interface ImportOptions {
dryRun?: boolean; // If true, validate but don't actually import
preventClash?: boolean; // If true, rename conflicting IDs
sanitize?: boolean; // If true, attempt to repair invalid data
}
/**
* Generic import function for any vendor data
*/
export async function importVendorData(
file: File,
parseFile: (file: File) => Promise<any>,
transformToConversations: (data: any) => Promise<any>,
vendorId: string,
options: ImportOptions = {},
): Promise<ImportResult> {
const {
dryRun = false,
preventClash = true,
sanitize = true,
} = options;
// Initialize result
const context: ImportContext = {
fileName: file.name,
fileSize: file.size,
fileHash: await calculateFileHash(file),
importedAt: Date.now(),
vendorId,
};
const result: ImportResult = {
success: false,
conversations: [],
warnings: [],
errors: [],
stats: {
conversationsImported: 0,
messagesImported: 0,
foldersImported: 0,
charactersImported: 0,
unsupportedItemsSkipped: 0,
},
context,
};
try {
// Step 1: Parse the file
const parseResult = await parseFile(file);
if (!parseResult.success || !parseResult.data) {
result.errors.push({
type: 'parse-error',
message: parseResult.error || 'Failed to parse file',
fatal: true,
});
return result;
}
result.warnings.push(...parseResult.warnings);
// Step 2: Transform to conversations
const transformResult = await transformToConversations(parseResult.data);
if (!transformResult.conversations || transformResult.conversations.length === 0) {
result.errors.push({
type: 'transform-error',
message: 'No conversations found in file',
fatal: true,
});
return result;
}
let conversations = transformResult.conversations;
result.warnings.push(...transformResult.warnings);
if (transformResult.unsupportedFeatures?.length > 0) {
result.warnings.push({
type: 'unsupported-feature',
message: `Unsupported features: ${transformResult.unsupportedFeatures.join(', ')}`,
});
result.stats.unsupportedItemsSkipped = transformResult.unsupportedFeatures.length;
}
// Step 3: Sanitize if requested
if (sanitize) {
conversations = conversations.map(sanitizeConversation);
}
// Step 4: Validate
const validationResult = validateConversations(conversations);
result.warnings.push(...validationResult.warnings);
result.errors.push(...validationResult.errors);
if (!validationResult.valid) {
result.errors.push({
type: 'validation-error',
message: 'Validation failed - see errors above',
fatal: true,
});
return result;
}
// Step 5: Resolve conflicts
if (preventClash) {
const existingConversations = useChatStore.getState().conversations;
conversations = resolveAllConflictsByRename(conversations, existingConversations);
}
// Step 6: Attach lineage metadata
for (const conversation of conversations) {
const originalId = (conversation.metadata as any)?.typingmind?.chatId || conversation.id;
attachLineage(conversation, context, originalId);
}
// Step 7: Calculate stats
result.stats.conversationsImported = conversations.length;
result.stats.messagesImported = conversations.reduce(
(sum: number, conv: DConversation) => sum + conv.messages.length,
0,
);
result.stats.charactersImported = conversations.reduce(
(sum: number, conv: DConversation) => sum + conv.messages.reduce(
(msgSum: number, msg) => msgSum + msg.fragments.reduce(
(fragSum: number, frag) => {
const text = (frag as any).part?.text;
return fragSum + (typeof text === 'string' ? text.length : 0);
},
0,
),
0,
),
0,
);
// Step 8: Import (if not dry run)
if (!dryRun) {
const chatStore = useChatStore.getState();
for (const conversation of conversations) {
chatStore.importConversation(conversation, preventClash);
}
}
result.conversations = conversations;
result.success = true;
return result;
} catch (error) {
result.errors.push({
type: 'parse-error',
message: error instanceof Error ? error.message : 'Unknown error',
fatal: true,
});
return result;
}
}
+121
View File
@@ -0,0 +1,121 @@
/**
* Data lineage and provenance tracking for imported conversations
* Provides import history, re-import detection, and file hash tracking
*/
import type { DConversation } from '~/common/stores/chat/chat.conversation';
import type { ImportContext } from './data.types';
// Lineage metadata stored in conversation metadata
export interface DConversationLineage {
// Import provenance
importSource: {
vendorId: string; // 'typingmind', 'chatgpt', etc.
fileName: string; // Original filename
fileHash: string; // SHA-256 hash of the file
importedAt: number; // Unix timestamp
};
// Re-import tracking
reimportCount?: number; // Number of times this was reimported
lastReimportAt?: number; // Last reimport timestamp
// Original IDs (for conflict resolution)
originalIds?: {
conversationId?: string;
chatId?: string;
folderId?: string;
};
}
/**
* Calculate SHA-256 hash of a file
*/
export async function calculateFileHash(file: File): Promise<string> {
const buffer = await file.arrayBuffer();
const hashBuffer = await crypto.subtle.digest('SHA-256', buffer);
const hashArray = Array.from(new Uint8Array(hashBuffer));
const hashHex = hashArray.map(b => b.toString(16).padStart(2, '0')).join('');
return hashHex;
}
/**
* Attach lineage metadata to a conversation
*/
export function attachLineage(
conversation: DConversation,
context: ImportContext,
originalId?: string,
): void {
const lineage: DConversationLineage = {
importSource: {
vendorId: context.vendorId,
fileName: context.fileName,
fileHash: context.fileHash,
importedAt: context.importedAt,
},
};
if (originalId) {
lineage.originalIds = {
conversationId: originalId,
};
}
// Store in conversation metadata (extending the DConversation type)
if (!(conversation as any).metadata) {
(conversation as any).metadata = {};
}
(conversation as any).metadata.lineage = lineage;
}
/**
* Get lineage metadata from a conversation
*/
export function getLineage(conversation: DConversation): DConversationLineage | null {
return (conversation as any).metadata?.lineage || null;
}
/**
* Check if a conversation was imported from a specific file hash
*/
export function isFromFile(conversation: DConversation, fileHash: string): boolean {
const lineage = getLineage(conversation);
return lineage?.importSource.fileHash === fileHash;
}
/**
* Detect if this is a re-import of existing conversations
*/
export function detectReimport(
existingConversations: DConversation[],
fileHash: string,
): {
isReimport: boolean;
existingConversations: DConversation[];
} {
const matches = existingConversations.filter(c => isFromFile(c, fileHash));
return {
isReimport: matches.length > 0,
existingConversations: matches,
};
}
/**
* Update lineage for a re-import
*/
export function updateLineageForReimport(conversation: DConversation): void {
const lineage = getLineage(conversation);
if (!lineage) return;
lineage.reimportCount = (lineage.reimportCount || 0) + 1;
lineage.lastReimportAt = Date.now();
}
+74
View File
@@ -0,0 +1,74 @@
/**
* Core types for the data import system
*/
import type { DConversation } from '~/common/stores/chat/chat.conversation';
// Import Source
export type ImportSourceId = string;
export interface ImportSource {
readonly id: ImportSourceId;
readonly label: string;
readonly description?: string;
readonly vendorId: string;
}
// Import Context
export interface ImportContext {
fileName: string;
fileSize: number;
fileHash: string;
importedAt: number;
vendorId: string;
}
// Import Result
export interface ImportResult {
success: boolean;
conversations: DConversation[];
warnings: ImportWarning[];
errors: ImportError[];
stats: ImportStats;
context: ImportContext;
}
export interface ImportWarning {
type: 'unsupported-feature' | 'data-loss' | 'approximation' | 'missing-data';
message: string;
details?: string;
affectedConversationIds?: string[];
}
export interface ImportError {
type: 'parse-error' | 'validation-error' | 'transform-error' | 'conflict-error';
message: string;
details?: string;
fatal: boolean;
}
export interface ImportStats {
conversationsImported: number;
messagesImported: number;
foldersImported: number;
charactersImported: number;
unsupportedItemsSkipped: number;
}
// Conflict Resolution
export type ConflictResolutionStrategy = 'skip' | 'rename' | 'overwrite' | 'merge';
export interface ConflictInfo {
type: 'conversation-id' | 'folder-id';
existingId: string;
proposedId: string;
resolution: ConflictResolutionStrategy;
}
+59
View File
@@ -0,0 +1,59 @@
/**
* Data import module
* Provides a modular system for importing conversations from various sources
*/
// Core types
export type {
ImportSourceId,
ImportSource,
ImportContext,
ImportResult,
ImportWarning,
ImportError,
ImportStats,
ConflictResolutionStrategy,
ConflictInfo,
} from './data.types';
// Lineage tracking
export {
calculateFileHash,
attachLineage,
getLineage,
isFromFile,
detectReimport,
updateLineageForReimport,
} from './data.lineage';
export type { DConversationLineage } from './data.lineage';
// Import orchestration
export { importVendorData } from './data.import';
export type { ImportOptions } from './data.import';
// Vendor types
export type {
VendorId,
IDataVendor,
ParseResult,
TransformResult,
ValidationResult,
} from './vendors/vendor.types';
// Vendor registry
export {
registerDataVendor,
getDataVendor,
getAllDataVendors,
hasDataVendor,
} from './vendors/vendor.registry';
// UI components
export { DataImportModal } from './ui/DataImportModal';
export { ImportConfirmStep } from './ui/ImportConfirmStep';
export { ImportResultModal } from './ui/ImportResultModal';
// TypingMind vendor
export { importTypingMindData } from './vendors/typingmind/typingmind.import-function';
export { TypingMindImporter } from './vendors/typingmind/typingmind.importer';
@@ -0,0 +1,143 @@
/**
* Conflict resolution for imports
* Handles ID conflicts and provides resolution strategies
*/
import { agiUuid } from '~/common/util/idUtils';
import type { DConversation } from '~/common/stores/chat/chat.conversation';
import type { ConflictInfo, ConflictResolutionStrategy } from '../data.types';
/**
* Detect conflicts between imported and existing conversations
*/
export function detectConflicts(
importedConversations: DConversation[],
existingConversations: DConversation[],
): ConflictInfo[] {
const conflicts: ConflictInfo[] = [];
const existingIds = new Set(existingConversations.map(c => c.id));
for (const imported of importedConversations) {
if (existingIds.has(imported.id)) {
conflicts.push({
type: 'conversation-id',
existingId: imported.id,
proposedId: imported.id,
resolution: 'rename', // Default strategy
});
}
}
return conflicts;
}
/**
* Resolve conflicts using the specified strategy
*/
export function resolveConflicts(
importedConversations: DConversation[],
conflicts: ConflictInfo[],
strategy: ConflictResolutionStrategy = 'rename',
): DConversation[] {
if (conflicts.length === 0) {
return importedConversations;
}
const conflictMap = new Map(conflicts.map(c => [c.existingId, c]));
return importedConversations.map(conversation => {
const conflict = conflictMap.get(conversation.id);
if (!conflict) {
return conversation;
}
switch (strategy) {
case 'rename':
return {
...conversation,
id: agiUuid('chat-dconversation'),
autoTitle: conversation.autoTitle
? `${conversation.autoTitle} (imported)`
: conversation.userTitle
? `${conversation.userTitle} (imported)`
: 'Imported conversation',
};
case 'skip':
// Mark for filtering later
return null as any;
case 'overwrite':
// Keep the original ID, will overwrite
return conversation;
case 'merge':
// For now, same as rename
// TODO: Implement actual merge logic
return {
...conversation,
id: agiUuid('chat-dconversation'),
};
default:
return conversation;
}
}).filter(Boolean);
}
/**
* Check if a conversation ID already exists
*/
export function hasConflict(
conversationId: string,
existingConversations: DConversation[],
): boolean {
return existingConversations.some(c => c.id === conversationId);
}
/**
* Generate a safe ID that doesn't conflict
*/
export function generateSafeId(
existingConversations: DConversation[],
): string {
const existingIds = new Set(existingConversations.map(c => c.id));
let newId: string;
let attempts = 0;
do {
newId = agiUuid('chat-dconversation');
attempts++;
} while (existingIds.has(newId) && attempts < 100);
return newId;
}
/**
* Resolve all conversation ID conflicts by renaming
*/
export function resolveAllConflictsByRename(
importedConversations: DConversation[],
existingConversations: DConversation[],
): DConversation[] {
const existingIds = new Set(existingConversations.map(c => c.id));
return importedConversations.map(conversation => {
if (existingIds.has(conversation.id)) {
return {
...conversation,
id: generateSafeId(existingConversations),
autoTitle: conversation.autoTitle
? `${conversation.autoTitle} (imported)`
: 'Imported conversation',
};
}
return conversation;
});
}
@@ -0,0 +1,159 @@
/**
* Transform utilities for converting vendor data to Big-AGI format
*/
import { agiUuid } from '~/common/util/idUtils';
import { createTextContentFragment } from '~/common/stores/chat/chat.fragments';
import type { DMessage, DMessageRole } from '~/common/stores/chat/chat.message';
import type { DConversation } from '~/common/stores/chat/chat.conversation';
import { defaultSystemPurposeId } from '../../../data';
/**
* Convert ISO timestamp string to Unix milliseconds
*/
export function isoToUnixMs(isoString: string | number): number {
if (typeof isoString === 'number') {
// Already a number - check if it's seconds or milliseconds
return isoString < 10000000000 ? isoString * 1000 : isoString;
}
try {
const timestamp = new Date(isoString).getTime();
return Number.isFinite(timestamp) ? timestamp : Date.now();
} catch {
return Date.now();
}
}
/**
* Normalize message content to text string
* Handles both string and array formats
*/
export function normalizeMessageContent(content: unknown): string {
// Already a string
if (typeof content === 'string') {
return content;
}
// Array of content parts
if (Array.isArray(content)) {
return content
.map(part => {
if (typeof part === 'string') return part;
if (part && typeof part === 'object' && 'text' in part) {
return String(part.text);
}
if (part && typeof part === 'object' && 'type' in part && part.type === 'text' && 'text' in part) {
return String(part.text);
}
return '';
})
.filter(text => text.length > 0)
.join('\n\n');
}
// Unknown format
return '';
}
/**
* Create a DMessage from basic components
*/
export function createImportedMessage(
role: DMessageRole,
content: string,
originalId?: string,
createdAt?: string | number,
): DMessage {
const message: DMessage = {
id: originalId || agiUuid('chat-dmessage'),
role,
fragments: [createTextContentFragment(content)],
tokenCount: 0,
created: createdAt ? isoToUnixMs(createdAt) : Date.now(),
updated: null,
};
return message;
}
/**
* Create an empty DConversation with defaults
*/
export function createImportedConversation(
title?: string,
originalId?: string,
createdAt?: string | number,
updatedAt?: string | number,
): DConversation {
const now = Date.now();
const conversation: DConversation = {
id: originalId || agiUuid('chat-dconversation'),
messages: [],
systemPurposeId: defaultSystemPurposeId,
tokenCount: 0,
created: createdAt ? isoToUnixMs(createdAt) : now,
updated: updatedAt ? isoToUnixMs(updatedAt) : now,
_abortController: null,
};
if (title) {
conversation.autoTitle = title;
}
return conversation;
}
/**
* Generate a unique ID that won't conflict with existing IDs
*/
export function generateNonConflictingId(
existingIds: Set<string>,
prefix: string = 'imported',
): string {
let attempt = 0;
let newId: string;
do {
newId = attempt === 0
? agiUuid('chat-dconversation')
: `${prefix}-${Date.now()}-${attempt}`;
attempt++;
} while (existingIds.has(newId) && attempt < 100);
if (attempt >= 100) {
// Fallback: use timestamp + random
newId = `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 11)}`;
}
return newId;
}
/**
* Deduplicate messages by ID, keeping first occurrence
*/
export function deduplicateMessages(messages: DMessage[]): DMessage[] {
const seen = new Set<string>();
return messages.filter(msg => {
if (seen.has(msg.id)) {
return false;
}
seen.add(msg.id);
return true;
});
}
/**
* Sort messages by creation timestamp
*/
export function sortMessagesByTimestamp(messages: DMessage[]): DMessage[] {
return [...messages].sort((a, b) => a.created - b.created);
}
@@ -0,0 +1,201 @@
/**
* Validation utilities for imported data
*/
import type { DConversation } from '~/common/stores/chat/chat.conversation';
import type { DMessage } from '~/common/stores/chat/chat.message';
import type { ImportError, ImportWarning } from '../data.types';
export interface ValidationResult {
valid: boolean;
errors: ImportError[];
warnings: ImportWarning[];
}
/**
* Validate a conversation structure
*/
export function validateConversation(conversation: DConversation): ValidationResult {
const errors: ImportError[] = [];
const warnings: ImportWarning[] = [];
// Required fields
if (!conversation.id) {
errors.push({
type: 'validation-error',
message: 'Conversation missing required ID',
fatal: true,
});
}
if (!conversation.messages) {
errors.push({
type: 'validation-error',
message: 'Conversation missing messages array',
fatal: true,
});
}
// Validate timestamps
if (conversation.created && (conversation.created < 0 || !Number.isFinite(conversation.created))) {
warnings.push({
type: 'data-loss',
message: 'Invalid created timestamp, using current time',
affectedConversationIds: [conversation.id],
});
}
// Validate messages
if (conversation.messages && Array.isArray(conversation.messages)) {
for (let i = 0; i < conversation.messages.length; i++) {
const messageResult = validateMessage(conversation.messages[i], i);
errors.push(...messageResult.errors);
warnings.push(...messageResult.warnings);
}
}
return {
valid: errors.filter(e => e.fatal).length === 0,
errors,
warnings,
};
}
/**
* Validate a message structure
*/
export function validateMessage(message: DMessage, index: number): ValidationResult {
const errors: ImportError[] = [];
const warnings: ImportWarning[] = [];
// Required fields
if (!message.id) {
errors.push({
type: 'validation-error',
message: `Message at index ${index} missing required ID`,
fatal: true,
});
}
if (!message.role || !['user', 'assistant', 'system'].includes(message.role)) {
errors.push({
type: 'validation-error',
message: `Message at index ${index} has invalid role: ${message.role}`,
fatal: true,
});
}
if (!message.fragments || !Array.isArray(message.fragments)) {
errors.push({
type: 'validation-error',
message: `Message at index ${index} missing fragments array`,
fatal: true,
});
}
// Validate timestamps
if (message.created && (message.created < 0 || !Number.isFinite(message.created))) {
warnings.push({
type: 'data-loss',
message: `Message at index ${index} has invalid created timestamp`,
});
}
return {
valid: errors.filter(e => e.fatal).length === 0,
errors,
warnings,
};
}
/**
* Validate all conversations in a batch
*/
export function validateConversations(conversations: DConversation[]): ValidationResult {
const allErrors: ImportError[] = [];
const allWarnings: ImportWarning[] = [];
for (const conversation of conversations) {
const result = validateConversation(conversation);
allErrors.push(...result.errors);
allWarnings.push(...result.warnings);
}
return {
valid: allErrors.filter(e => e.fatal).length === 0,
errors: allErrors,
warnings: allWarnings,
};
}
/**
* Sanitize and repair a conversation to make it valid
*/
export function sanitizeConversation(conversation: DConversation): DConversation {
// Ensure required fields
if (!conversation.id) {
conversation.id = `imported-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`;
}
if (!conversation.messages) {
conversation.messages = [];
}
if (!conversation.created || conversation.created < 0) {
conversation.created = Date.now();
}
if (!conversation.updated) {
conversation.updated = conversation.created;
}
if (typeof conversation.tokenCount !== 'number') {
conversation.tokenCount = 0;
}
// Sanitize messages
conversation.messages = conversation.messages
.map((msg, index) => sanitizeMessage(msg, index))
.filter(msg => msg !== null) as DMessage[];
return conversation;
}
/**
* Sanitize and repair a message to make it valid
*/
function sanitizeMessage(message: DMessage, index: number): DMessage | null {
// Skip messages with fatal errors
if (!message || typeof message !== 'object') {
return null;
}
// Ensure required fields
if (!message.id) {
message.id = `imported-msg-${Date.now()}-${index}`;
}
if (!message.role || !['user', 'assistant', 'system'].includes(message.role)) {
message.role = 'user'; // Default to user
}
if (!message.fragments || !Array.isArray(message.fragments)) {
message.fragments = [];
}
if (!message.created || message.created < 0) {
message.created = Date.now();
}
if (typeof message.tokenCount !== 'number') {
message.tokenCount = 0;
}
return message;
}
+80
View File
@@ -0,0 +1,80 @@
/**
* tRPC router for data import operations
* Provides server-side endpoints for data import if needed
*/
import * as z from 'zod/v4';
import { createTRPCRouter, publicProcedure } from '~/server/trpc/trpc.server';
/**
* Data import router
* Currently minimal - most import operations happen client-side
*/
export const dataRouter = createTRPCRouter({
/**
* Validate a data import file (server-side validation)
* This can be used for large files or security checks
*/
validateImportFile: publicProcedure
.input(z.object({
vendorId: z.enum(['typingmind', 'chatgpt', 'bigagi']),
fileSize: z.number(),
fileHash: z.string(),
}))
.output(z.object({
valid: z.boolean(),
message: z.string().optional(),
}))
.query(async ({ input }: { input: { vendorId: string; fileSize: number; fileHash: string } }) => {
// Basic validation
const MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
if (input.fileSize > MAX_FILE_SIZE) {
return {
valid: false,
message: `File too large: ${(input.fileSize / 1024 / 1024).toFixed(2)}MB (max ${MAX_FILE_SIZE / 1024 / 1024}MB)`,
};
}
if (input.fileSize === 0) {
return {
valid: false,
message: 'File is empty',
};
}
// All checks passed
return {
valid: true,
};
}),
/**
* Get import statistics
* Returns information about previous imports if tracked
*/
getImportStats: publicProcedure
.output(z.object({
totalImports: z.number(),
recentImports: z.array(z.object({
vendorId: z.string(),
fileName: z.string(),
conversationCount: z.number(),
importedAt: z.number(),
})),
}))
.query(async () => {
// For now, return empty stats
// This could be expanded to track import history in the future
return {
totalImports: 0,
recentImports: [],
};
}),
});
export type DataRouter = typeof dataRouter;
+226
View File
@@ -0,0 +1,226 @@
/**
* Main data import modal component
* Provides a multi-step import flow: file selection -> validation -> confirmation -> import
*/
import * as React from 'react';
import { Box, Button, CircularProgress, Typography } from '@mui/joy';
import FileUploadIcon from '@mui/icons-material/FileUpload';
import { GoodModal } from '~/common/components/modals/GoodModal';
import type { DConversationId } from '~/common/stores/chat/chat.conversation';
import type { VendorId } from '../vendors/vendor.types';
import type { ImportResult } from '../data.types';
import { ImportConfirmStep } from './ImportConfirmStep';
import { ImportResultModal } from './ImportResultModal';
interface DataImportModalProps {
vendorId: VendorId;
vendorLabel: string;
open: boolean;
onConversationActivate?: (conversationId: DConversationId) => void;
onClose: () => void;
}
type ImportStep = 'select' | 'processing' | 'confirm' | 'importing' | 'complete';
export function DataImportModal(props: DataImportModalProps) {
const { vendorId, vendorLabel, open, onClose } = props;
// State
const [step, setStep] = React.useState<ImportStep>('select');
const [selectedFile, setSelectedFile] = React.useState<File | null>(null);
const [importPreview, setImportPreview] = React.useState<any>(null);
const [importResult, setImportResult] = React.useState<ImportResult | null>(null);
const [error, setError] = React.useState<string | null>(null);
// File input ref
const fileInputRef = React.useRef<HTMLInputElement>(null);
const handleFileSelect = async (event: React.ChangeEvent<HTMLInputElement>) => {
const file = event.target.files?.[0];
if (!file) return;
setSelectedFile(file);
setStep('processing');
setError(null);
try {
// Dynamic import of the vendor importer
const { importTypingMindData } = await import('../vendors/typingmind/typingmind.import-function');
// TODO: Use vendor registry to get the correct importer
// For now, only TypingMind is supported
if (vendorId !== 'typingmind') {
throw new Error(`Vendor ${vendorId} not yet supported`);
}
// Parse and validate the file
const result = await importTypingMindData(file, { dryRun: true });
if (!result.success) {
setError(result.errors[0]?.message || 'Failed to parse file');
setStep('select');
return;
}
setImportPreview(result);
setStep('confirm');
} catch (err) {
setError(err instanceof Error ? err.message : 'Unknown error');
setStep('select');
}
};
const handleConfirmImport = async () => {
if (!selectedFile) return;
setStep('importing');
setError(null);
try {
// Dynamic import of the vendor importer
const { importTypingMindData } = await import('../vendors/typingmind/typingmind.import-function');
// Actually perform the import
const result = await importTypingMindData(selectedFile, { dryRun: false });
setImportResult(result);
setStep('complete');
// Activate the last imported conversation
if (result.success && result.conversations.length > 0 && props.onConversationActivate) {
const lastConv = result.conversations[result.conversations.length - 1];
props.onConversationActivate(lastConv.id);
}
} catch (err) {
setError(err instanceof Error ? err.message : 'Unknown error');
setStep('confirm');
}
};
const handleCancelImport = () => {
setStep('select');
setSelectedFile(null);
setImportPreview(null);
setError(null);
};
const handleCloseComplete = () => {
setStep('select');
setSelectedFile(null);
setImportPreview(null);
setImportResult(null);
setError(null);
onClose();
};
const handleOpenFilePicker = () => {
fileInputRef.current?.click();
};
// Render step content
const renderStepContent = () => {
switch (step) {
case 'select':
return (
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2, alignItems: 'center', py: 4 }}>
<Typography level='body-md'>
Select a {vendorLabel} export file to import
</Typography>
<input
ref={fileInputRef}
type='file'
accept='.json,application/json'
onChange={handleFileSelect}
style={{ display: 'none' }}
/>
<Button
variant='solid'
color='primary'
size='lg'
startDecorator={<FileUploadIcon />}
onClick={handleOpenFilePicker}
>
Choose File
</Button>
{error && (
<Typography level='body-sm' color='danger'>
{error}
</Typography>
)}
</Box>
);
case 'processing':
return (
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2, alignItems: 'center', py: 4 }}>
<CircularProgress />
<Typography level='body-md'>Processing file...</Typography>
</Box>
);
case 'confirm':
return importPreview ? (
<ImportConfirmStep
preview={importPreview}
vendorLabel={vendorLabel}
onConfirm={handleConfirmImport}
onCancel={handleCancelImport}
/>
) : null;
case 'importing':
return (
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2, alignItems: 'center', py: 4 }}>
<CircularProgress />
<Typography level='body-md'>Importing conversations...</Typography>
</Box>
);
case 'complete':
return null; // Handled by separate modal
default:
return null;
}
};
// Show result modal when complete
if (step === 'complete' && importResult) {
return (
<ImportResultModal
result={importResult}
vendorLabel={vendorLabel}
onClose={handleCloseComplete}
/>
);
}
return (
<GoodModal
open={open && step !== 'complete'}
title={`Import from ${vendorLabel}`}
onClose={step === 'processing' || step === 'importing' ? undefined : onClose}
>
<Box>{renderStepContent()}</Box>
</GoodModal>
);
}
+107
View File
@@ -0,0 +1,107 @@
/**
* Confirmation step for data import
* Shows preview of what will be imported with warnings
*/
import * as React from 'react';
import { Alert, Box, Button, Divider, List, ListItem, Typography } from '@mui/joy';
import WarningIcon from '@mui/icons-material/Warning';
import InfoIcon from '@mui/icons-material/Info';
interface ImportConfirmStepProps {
preview: any;
vendorLabel: string;
onConfirm: () => void;
onCancel: () => void;
}
export function ImportConfirmStep(props: ImportConfirmStepProps) {
const { preview, vendorLabel, onConfirm, onCancel } = props;
const conversationCount = preview.stats?.conversationsImported || 0;
const messageCount = preview.stats?.messagesImported || 0;
const hasWarnings = preview.warnings?.length > 0;
const hasErrors = preview.errors?.length > 0;
return (
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
{/* Summary */}
<Alert variant='soft' color='primary' startDecorator={<InfoIcon />}>
<Box>
<Typography level='title-sm'>Ready to Import</Typography>
<Typography level='body-sm'>
{conversationCount} conversation{conversationCount !== 1 ? 's' : ''} with {messageCount} message{messageCount !== 1 ? 's' : ''}
</Typography>
</Box>
</Alert>
{/* Warnings */}
{hasWarnings && (
<>
<Alert variant='soft' color='warning' startDecorator={<WarningIcon />}>
<Typography level='title-sm'>Warnings</Typography>
</Alert>
<List size='sm'>
{preview.warnings.slice(0, 5).map((warning: any, idx: number) => (
<ListItem key={idx}>
<Typography level='body-sm'>
{warning.message}
</Typography>
</ListItem>
))}
{preview.warnings.length > 5 && (
<ListItem>
<Typography level='body-sm' fontStyle='italic'>
... and {preview.warnings.length - 5} more warning{preview.warnings.length - 5 !== 1 ? 's' : ''}
</Typography>
</ListItem>
)}
</List>
</>
)}
{/* Errors */}
{hasErrors && (
<>
<Alert variant='soft' color='danger'>
<Typography level='title-sm'>Errors</Typography>
</Alert>
<List size='sm'>
{preview.errors.map((error: any, idx: number) => (
<ListItem key={idx}>
<Typography level='body-sm' color='danger'>
{error.message}
</Typography>
</ListItem>
))}
</List>
</>
)}
<Divider />
{/* Actions */}
<Box sx={{ display: 'flex', gap: 1, justifyContent: 'flex-end' }}>
<Button
variant='plain'
color='neutral'
onClick={onCancel}
>
Cancel
</Button>
<Button
variant='solid'
color='primary'
onClick={onConfirm}
disabled={hasErrors}
>
Import {conversationCount} Conversation{conversationCount !== 1 ? 's' : ''}
</Button>
</Box>
</Box>
);
}
+146
View File
@@ -0,0 +1,146 @@
/**
* Display results after import completion
*/
import * as React from 'react';
import { Alert, Box, Divider, List, ListItem, Typography } from '@mui/joy';
import CheckCircleIcon from '@mui/icons-material/CheckCircle';
import WarningIcon from '@mui/icons-material/Warning';
import ErrorIcon from '@mui/icons-material/Error';
import { GoodModal } from '~/common/components/modals/GoodModal';
import type { ImportResult } from '../data.types';
interface ImportResultModalProps {
result: ImportResult;
vendorLabel: string;
onClose: () => void;
}
export function ImportResultModal(props: ImportResultModalProps) {
const { result, vendorLabel, onClose } = props;
const { success, stats, warnings, errors } = result;
const hasWarnings = warnings.length > 0;
const hasErrors = errors.length > 0;
return (
<GoodModal
open
title={success ? 'Import Successful' : 'Import Failed'}
strongerTitle
onClose={onClose}
>
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
<Divider />
{/* Success summary */}
{success && (
<Alert variant='soft' color='success' startDecorator={<CheckCircleIcon />}>
<Box>
<Typography level='title-sm'>Import Complete</Typography>
<Typography level='body-sm'>
Imported {stats.conversationsImported} conversation{stats.conversationsImported !== 1 ? 's' : ''} with {stats.messagesImported} message{stats.messagesImported !== 1 ? 's' : ''}
</Typography>
</Box>
</Alert>
)}
{/* Error summary */}
{!success && (
<Alert variant='soft' color='danger' startDecorator={<ErrorIcon />}>
<Box>
<Typography level='title-sm'>Import Failed</Typography>
<Typography level='body-sm'>
{errors[0]?.message || 'Unknown error occurred'}
</Typography>
</Box>
</Alert>
)}
{/* Statistics */}
{success && (
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 1 }}>
<Typography level='title-sm'>Import Statistics</Typography>
<List size='sm'>
<ListItem>Conversations: {stats.conversationsImported}</ListItem>
<ListItem>Messages: {stats.messagesImported}</ListItem>
{stats.charactersImported > 0 && (
<ListItem>Characters: {stats.charactersImported.toLocaleString()}</ListItem>
)}
{stats.unsupportedItemsSkipped > 0 && (
<ListItem>Unsupported items skipped: {stats.unsupportedItemsSkipped}</ListItem>
)}
</List>
</Box>
)}
{/* Warnings */}
{hasWarnings && (
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 1 }}>
<Alert variant='soft' color='warning' startDecorator={<WarningIcon />}>
<Typography level='title-sm'>
{warnings.length} Warning{warnings.length !== 1 ? 's' : ''}
</Typography>
</Alert>
<List size='sm'>
{warnings.slice(0, 10).map((warning, idx) => (
<ListItem key={idx}>
<Typography level='body-sm'>
[{warning.type}] {warning.message}
</Typography>
</ListItem>
))}
{warnings.length > 10 && (
<ListItem>
<Typography level='body-sm' fontStyle='italic'>
... and {warnings.length - 10} more warning{warnings.length - 10 !== 1 ? 's' : ''}
</Typography>
</ListItem>
)}
</List>
</Box>
)}
{/* Errors (non-fatal) */}
{hasErrors && success && (
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 1 }}>
<Alert variant='soft' color='danger' startDecorator={<ErrorIcon />}>
<Typography level='title-sm'>
{errors.length} Error{errors.length !== 1 ? 's' : ''}
</Typography>
</Alert>
<List size='sm'>
{errors.slice(0, 5).map((error, idx) => (
<ListItem key={idx}>
<Typography level='body-sm' color='danger'>
[{error.type}] {error.message}
</Typography>
</ListItem>
))}
{errors.length > 5 && (
<ListItem>
<Typography level='body-sm' fontStyle='italic'>
... and {errors.length - 5} more error{errors.length - 5 !== 1 ? 's' : ''}
</Typography>
</ListItem>
)}
</List>
</Box>
)}
{/* Information */}
<Typography level='body-sm' sx={{ mt: 2 }}>
{success
? `The imported conversations are now available in your chat list. ${stats.conversationsImported > 1 ? 'The most recent conversation is' : 'It is'} now active.`
: 'Please check the error messages above and try again with a valid export file.'
}
</Typography>
</Box>
</GoodModal>
);
}
+18
View File
@@ -0,0 +1,18 @@
/**
* TypingMind vendor exports
*/
export { TypingMindImporter } from './typingmind.importer';
export { importTypingMindData } from './typingmind.import-function';
export { parseTypingMindFile, validateTypingMindFile } from './typingmind.parser';
export { transformTypingMindToConversations, getTransformStats } from './typingmind.transformer';
export { validateTypingMindSource } from './typingmind.validator';
export type {
TypingMindExport,
TypingMindChat,
TypingMindMessage,
TypingMindFolder,
TypingMindUserPrompt,
TypingMindUserCharacter,
} from './typingmind.schema';
@@ -0,0 +1,27 @@
/**
* High-level import function for TypingMind data
* This is the main entry point for TypingMind imports
*/
import type { ImportResult } from '../../data.types';
import type { ImportOptions } from '../../data.import';
import { importVendorData } from '../../data.import';
import { parseTypingMindFile } from './typingmind.parser';
import { transformTypingMindToConversations } from './typingmind.transformer';
/**
* Import TypingMind data from a file
*/
export async function importTypingMindData(
file: File,
options: ImportOptions = {},
): Promise<ImportResult> {
return importVendorData(
file,
parseTypingMindFile,
transformTypingMindToConversations,
'typingmind',
options,
);
}
@@ -0,0 +1,42 @@
/**
* TypingMind importer - main vendor implementation
*/
import type { IDataVendor, ParseResult, TransformResult, ValidationResult } from '../vendor.types';
import type { TypingMindExport } from './typingmind.schema';
import { parseTypingMindFile, validateTypingMindFile } from './typingmind.parser';
import { transformTypingMindToConversations } from './typingmind.transformer';
import { validateTypingMindSource } from './typingmind.validator';
/**
* TypingMind data import vendor
*/
export const TypingMindImporter: IDataVendor = {
id: 'typingmind',
label: 'TypingMind',
description: 'Import conversations from TypingMind export files',
capabilities: {
supportsFiles: true,
supportsUrl: false,
supportsText: false,
},
async validateFile(file: File): Promise<boolean> {
return validateTypingMindFile(file);
},
async parseFile(file: File): Promise<ParseResult> {
return parseTypingMindFile(file);
},
async transformToConversations(parsedData: TypingMindExport): Promise<TransformResult> {
return transformTypingMindToConversations(parsedData);
},
validateSource(data: TypingMindExport): ValidationResult {
return validateTypingMindSource(data);
},
};
+119
View File
@@ -0,0 +1,119 @@
/**
* Parser for TypingMind export files
*/
import type { ParseResult } from '../vendor.types';
import type { ImportWarning } from '../../data.types';
import { typingMindExportSchema, type TypingMindExport } from './typingmind.schema';
/**
* Parse a TypingMind export file
*/
export async function parseTypingMindFile(file: File): Promise<ParseResult> {
const warnings: ImportWarning[] = [];
try {
// Read file content
const fileContent = await file.text();
// Parse JSON
let jsonData: any;
try {
jsonData = JSON.parse(fileContent);
} catch (parseError) {
return {
success: false,
data: null,
warnings: [],
error: `Invalid JSON: ${parseError instanceof Error ? parseError.message : 'Unknown error'}`,
};
}
// Validate against schema
const parseResult = typingMindExportSchema.safeParse(jsonData);
if (!parseResult.success) {
return {
success: false,
data: null,
warnings: [],
error: `Schema validation failed: ${parseResult.error.message}`,
};
}
const data = parseResult.data;
// Add warnings for optional features
if (data.data.folders && data.data.folders.length > 0) {
warnings.push({
type: 'unsupported-feature',
message: `Found ${data.data.folders.length} folders - folder structure will be preserved`,
});
}
if (data.data.userPrompts && data.data.userPrompts.length > 0) {
warnings.push({
type: 'unsupported-feature',
message: `Found ${data.data.userPrompts.length} user prompts - these are not yet supported`,
});
}
if (data.data.userCharacters && data.data.userCharacters.length > 0) {
warnings.push({
type: 'unsupported-feature',
message: `Found ${data.data.userCharacters.length} user characters - these are not yet supported`,
});
}
return {
success: true,
data,
warnings,
};
} catch (error) {
return {
success: false,
data: null,
warnings: [],
error: `Failed to parse file: ${error instanceof Error ? error.message : 'Unknown error'}`,
};
}
}
/**
* Validate that a file appears to be a TypingMind export
*/
export async function validateTypingMindFile(file: File): Promise<boolean> {
try {
// Check file extension
if (!file.name.endsWith('.json')) {
return false;
}
// Check file size (must be reasonable)
if (file.size === 0 || file.size > 100 * 1024 * 1024) { // Max 100MB
return false;
}
// Quick check: try to parse and look for TypingMind structure
const content = await file.text();
const json = JSON.parse(content);
// Look for TypingMind-specific structure
return (
json &&
typeof json === 'object' &&
'data' in json &&
json.data &&
typeof json.data === 'object' &&
'chats' in json.data &&
Array.isArray(json.data.chats)
);
} catch {
return false;
}
}
+105
View File
@@ -0,0 +1,105 @@
/**
* Zod schemas for TypingMind export format
* Using relaxed parsing with .passthrough() to allow schema evolution
*/
import * as z from 'zod/v4';
/**
* Message content part (text, image, etc.)
*/
const typingMindContentPartSchema = z.object({
type: z.string(),
text: z.string().optional(),
image_url: z.any().optional(),
}).passthrough();
/**
* Message schema
* Content can be either a string or an array of content parts
*/
const typingMindMessageSchema = z.object({
role: z.enum(['user', 'assistant', 'system']),
content: z.union([
z.string(),
z.array(typingMindContentPartSchema),
]),
uuid: z.string().optional(),
createdAt: z.string().optional(),
updatedAt: z.string().optional(),
}).passthrough();
/**
* Chat/Conversation schema
*/
const typingMindChatSchema = z.object({
chatID: z.string(),
chatTitle: z.string().optional(),
messages: z.array(typingMindMessageSchema),
createdAt: z.string().optional(),
updatedAt: z.string().optional(),
folderID: z.string().optional(),
model: z.string().optional(),
}).passthrough();
/**
* Folder schema
*/
const typingMindFolderSchema = z.object({
id: z.string(),
name: z.string(),
color: z.string().optional(),
createdAt: z.string().optional(),
parentID: z.string().optional(),
}).passthrough();
/**
* User prompt schema (custom instructions)
*/
const typingMindUserPromptSchema = z.object({
id: z.string(),
name: z.string(),
prompt: z.string(),
createdAt: z.string().optional(),
}).passthrough();
/**
* User character schema (personas)
*/
const typingMindUserCharacterSchema = z.object({
id: z.string(),
name: z.string(),
description: z.string().optional(),
prompt: z.string().optional(),
createdAt: z.string().optional(),
}).passthrough();
/**
* Main TypingMind export schema
*/
export const typingMindExportSchema = z.object({
data: z.object({
chats: z.array(typingMindChatSchema),
folders: z.array(typingMindFolderSchema).optional(),
userPrompts: z.array(typingMindUserPromptSchema).optional(),
userCharacters: z.array(typingMindUserCharacterSchema).optional(),
}).passthrough(),
}).passthrough();
/**
* TypeScript types inferred from schemas
*/
export type TypingMindExport = z.infer<typeof typingMindExportSchema>;
export type TypingMindChat = z.infer<typeof typingMindChatSchema>;
export type TypingMindMessage = z.infer<typeof typingMindMessageSchema>;
export type TypingMindFolder = z.infer<typeof typingMindFolderSchema>;
export type TypingMindUserPrompt = z.infer<typeof typingMindUserPromptSchema>;
export type TypingMindUserCharacter = z.infer<typeof typingMindUserCharacterSchema>;
@@ -0,0 +1,166 @@
/**
* Transform TypingMind data to Big-AGI format
*/
import type { DConversation } from '~/common/stores/chat/chat.conversation';
import type { DMessage, DMessageRole } from '~/common/stores/chat/chat.message';
import type { TransformResult } from '../vendor.types';
import type { ImportWarning } from '../../data.types';
import type { TypingMindExport, TypingMindChat, TypingMindMessage } from './typingmind.schema';
import { createImportedConversation, createImportedMessage, normalizeMessageContent } from '../../pipeline/import.transformer';
import { deduplicateMessages, sortMessagesByTimestamp } from '../../pipeline/import.transformer';
/**
* Transform TypingMind export to Big-AGI conversations
*/
export async function transformTypingMindToConversations(
data: TypingMindExport,
): Promise<TransformResult> {
const warnings: ImportWarning[] = [];
const unsupportedFeatures: string[] = [];
const conversations: DConversation[] = [];
// Track unsupported features
if (data.data.userPrompts?.length) {
unsupportedFeatures.push('Custom user prompts');
}
if (data.data.userCharacters?.length) {
unsupportedFeatures.push('User characters/personas');
}
// Transform chats
for (const chat of data.data.chats) {
try {
const conversation = transformTypingMindChat(chat, warnings);
if (conversation && conversation.messages.length > 0) {
conversations.push(conversation);
} else {
warnings.push({
type: 'data-loss',
message: `Skipped empty chat: ${chat.chatTitle || chat.chatID}`,
});
}
} catch (error) {
warnings.push({
type: 'data-loss',
message: `Failed to transform chat ${chat.chatID}: ${error instanceof Error ? error.message : 'Unknown error'}`,
});
}
}
return {
conversations,
warnings,
unsupportedFeatures,
};
}
/**
* Transform a single TypingMind chat to a DConversation
*/
function transformTypingMindChat(
chat: TypingMindChat,
warnings: ImportWarning[],
): DConversation {
// Create conversation
const conversation = createImportedConversation(
chat.chatTitle || 'Untitled Chat',
chat.chatID,
chat.createdAt,
chat.updatedAt,
);
// Transform messages
const messages: DMessage[] = [];
for (const tmMessage of chat.messages) {
try {
const message = transformTypingMindMessage(tmMessage);
if (message) {
messages.push(message);
}
} catch (error) {
warnings.push({
type: 'data-loss',
message: `Failed to transform message in chat ${chat.chatID}`,
affectedConversationIds: [chat.chatID],
});
}
}
// Deduplicate and sort messages
conversation.messages = sortMessagesByTimestamp(deduplicateMessages(messages));
// Store folder reference in metadata if present
if (chat.folderID) {
if (!(conversation as any).metadata) {
(conversation as any).metadata = {};
}
(conversation as any).metadata.typingmind = {
folderId: chat.folderID,
model: chat.model,
};
}
return conversation;
}
/**
* Transform a single TypingMind message to a DMessage
*/
function transformTypingMindMessage(tmMessage: TypingMindMessage): DMessage | null {
// Normalize content
const content = normalizeMessageContent(tmMessage.content);
if (!content || content.trim().length === 0) {
return null;
}
// Map role
const role: DMessageRole = tmMessage.role === 'system'
? 'system'
: tmMessage.role === 'assistant'
? 'assistant'
: 'user';
// Create message
const message = createImportedMessage(
role,
content,
tmMessage.uuid,
tmMessage.createdAt,
);
return message;
}
/**
* Get statistics about the transform
*/
export function getTransformStats(result: TransformResult) {
const totalMessages = result.conversations.reduce(
(sum, conv) => sum + conv.messages.length,
0,
);
return {
conversationsImported: result.conversations.length,
messagesImported: totalMessages,
foldersImported: 0, // Not yet supported
charactersImported: result.conversations.reduce(
(sum, conv) => sum + conv.messages.reduce(
(msgSum, msg) => msgSum + msg.fragments.reduce(
(fragSum, frag) => fragSum + ((frag as any).part?.text?.length || 0),
0,
),
0,
),
0,
),
unsupportedItemsSkipped: result.unsupportedFeatures.length,
};
}
@@ -0,0 +1,98 @@
# TypingMind Import - Unsupported Features
This document lists features from TypingMind exports that are not currently supported in Big-AGI import.
## Currently Unsupported
### 1. User Prompts / Custom Instructions
- **TypingMind Feature**: Custom user prompts and instructions
- **Status**: Not imported
- **Workaround**: Manually recreate in Big-AGI settings or personas
- **Future**: May be mapped to Big-AGI personas system
### 2. User Characters / Personas
- **TypingMind Feature**: User-created characters with custom personalities
- **Status**: Not imported
- **Workaround**: Manually recreate using Big-AGI personas feature
- **Future**: Should map to Big-AGI personas system
### 3. Folder Hierarchy
- **TypingMind Feature**: Nested folder organization
- **Status**: Folder IDs are preserved in metadata but not displayed
- **Workaround**: Folder references are stored and can be used for future organization
- **Future**: Will be mapped to Big-AGI folder system
### 4. Model Preferences
- **TypingMind Feature**: Per-chat model selection
- **Status**: Model information is preserved in metadata but not applied
- **Workaround**: Manually select models in Big-AGI
- **Future**: May restore model preferences
### 5. Attachments / Images
- **TypingMind Feature**: Image and file attachments in messages
- **Status**: Image URLs are lost in content normalization
- **Workaround**: Images must be re-uploaded
- **Future**: Should support image reference preservation
### 6. Message Metadata
- **TypingMind Feature**: Token counts, costs, timing data
- **Status**: Not preserved
- **Workaround**: Big-AGI will recalculate token counts
- **Future**: May preserve historical metadata
## Partially Supported
### 1. Message Content
- **Text content**: Fully supported
- **Multi-part content**: Text parts are concatenated
- **Code blocks**: Preserved as-is
- **Formatting**: Markdown formatting preserved
### 2. Timestamps
- **Created timestamps**: Fully supported
- **Updated timestamps**: Converted to conversation update time
- **Timezone handling**: Preserved via ISO 8601 format
### 3. Message Roles
- **User messages**: Fully supported
- **Assistant messages**: Fully supported
- **System messages**: Fully supported
- **Function/tool messages**: Not present in TypingMind exports
## Fully Supported
### 1. Conversations
- Chat ID preservation
- Chat titles
- Creation and update timestamps
- Message history
### 2. Messages
- Role (user/assistant/system)
- Text content
- Message order
- UUIDs
### 3. Data Lineage
- Original file hash tracking
- Re-import detection
- Original ID preservation
## Migration Notes
When migrating from TypingMind to Big-AGI:
1. **Export your data** from TypingMind settings
2. **Review custom prompts** and recreate as Big-AGI personas
3. **Note folder structure** - consider recreating in Big-AGI
4. **Import the JSON file** - conversations will be preserved
5. **Review imported chats** - verify content and organization
## Reporting Issues
If you encounter issues with TypingMind imports:
1. Check that your export file is valid JSON
2. Ensure you're using the latest TypingMind export format
3. Review the import warnings and errors
4. Report issues with sample data (remove sensitive content)
@@ -0,0 +1,68 @@
/**
* Validator for TypingMind source data
*/
import type { ValidationResult } from '../vendor.types';
import type { TypingMindExport } from './typingmind.schema';
/**
* Validate TypingMind source data before transformation
*/
export function validateTypingMindSource(data: TypingMindExport): ValidationResult {
const errors: string[] = [];
const warnings: string[] = [];
// Check required structure
if (!data.data) {
errors.push('Missing data object');
return { valid: false, errors, warnings };
}
if (!data.data.chats || !Array.isArray(data.data.chats)) {
errors.push('Missing or invalid chats array');
return { valid: false, errors, warnings };
}
// Check if empty
if (data.data.chats.length === 0) {
warnings.push('No chats found in export');
}
// Validate individual chats
for (let i = 0; i < data.data.chats.length; i++) {
const chat = data.data.chats[i];
if (!chat.chatID) {
errors.push(`Chat at index ${i} is missing chatID`);
}
if (!chat.messages || !Array.isArray(chat.messages)) {
errors.push(`Chat ${chat.chatID || i} has invalid messages array`);
continue;
}
if (chat.messages.length === 0) {
warnings.push(`Chat ${chat.chatTitle || chat.chatID} is empty`);
}
// Validate messages
for (let j = 0; j < chat.messages.length; j++) {
const message = chat.messages[j];
if (!message.role) {
errors.push(`Message ${j} in chat ${chat.chatID} is missing role`);
}
if (!message.content) {
warnings.push(`Message ${j} in chat ${chat.chatID} has empty content`);
}
}
}
return {
valid: errors.length === 0,
errors,
warnings,
};
}
+47
View File
@@ -0,0 +1,47 @@
/**
* Registry for data import vendors
*/
import type { IDataVendor, VendorId } from './vendor.types';
/**
* Global registry of data import vendors
*/
const _vendorRegistry = new Map<VendorId, IDataVendor>();
/**
* Register a data import vendor
*/
export function registerDataVendor(vendor: IDataVendor): void {
if (_vendorRegistry.has(vendor.id)) {
console.warn(`[Data Import] Vendor ${vendor.id} is already registered`);
return;
}
_vendorRegistry.set(vendor.id, vendor);
}
/**
* Get a vendor by ID
*/
export function getDataVendor(vendorId: VendorId): IDataVendor | null {
return _vendorRegistry.get(vendorId) || null;
}
/**
* Get all registered vendors
*/
export function getAllDataVendors(): IDataVendor[] {
return Array.from(_vendorRegistry.values());
}
/**
* Check if a vendor is registered
*/
export function hasDataVendor(vendorId: VendorId): boolean {
return _vendorRegistry.has(vendorId);
}
+71
View File
@@ -0,0 +1,71 @@
/**
* Base types for data import vendors
*/
import type { DConversation } from '~/common/stores/chat/chat.conversation';
import type { ImportResult, ImportWarning } from '../data.types';
/**
* Vendor ID - unique identifier for each import source
*/
export type VendorId = 'typingmind' | 'chatgpt' | 'bigagi';
/**
* Base interface for all data import vendors
*/
export interface IDataVendor {
// Vendor metadata
readonly id: VendorId;
readonly label: string;
readonly description: string;
// Capabilities
readonly capabilities: {
supportsFiles: boolean;
supportsUrl: boolean;
supportsText: boolean;
};
// File validation
validateFile?(file: File): Promise<boolean>;
// Parse and transform
parseFile(file: File): Promise<ParseResult>;
transformToConversations(parsedData: any): Promise<TransformResult>;
// Optional: validate source data before transformation
validateSource?(data: any): ValidationResult;
}
/**
* Result of parsing a file
*/
export interface ParseResult {
success: boolean;
data: any;
warnings: ImportWarning[];
error?: string;
}
/**
* Result of transforming parsed data to conversations
*/
export interface TransformResult {
conversations: DConversation[];
warnings: ImportWarning[];
unsupportedFeatures: string[];
}
/**
* Result of validating source data
*/
export interface ValidationResult {
valid: boolean;
errors: string[];
warnings: string[];
}
+19
View File
@@ -2,6 +2,7 @@ import * as React from 'react';
import { Box, Button, FormControl, Input, Sheet, Textarea, Typography } from '@mui/joy';
import FileUploadIcon from '@mui/icons-material/FileUpload';
import CloudUploadIcon from '@mui/icons-material/CloudUpload';
import { Brand } from '~/common/app.config';
import { FormRadioOption } from '~/common/components/forms/FormRadioControl';
@@ -20,6 +21,7 @@ import { importConversationsFromFilesAtRest, openConversationsAtRestPicker } fro
import { FlashRestore } from './BackupRestore';
import { ImportedOutcome, ImportOutcomeModal } from './ImportOutcomeModal';
import { DataImportModal } from '~/modules/data/ui/DataImportModal';
export type ImportConfig = { dir: 'import' };
@@ -43,6 +45,7 @@ export function ImportChats(props: { onConversationActivate: (conversationId: DC
const [chatGptSource, setChatGptSource] = React.useState('');
const [importJson, setImportJson] = React.useState<string | null>(null);
const [importOutcome, setImportOutcome] = React.useState<ImportedOutcome | null>(null);
const [typingMindModalOpen, setTypingMindModalOpen] = React.useState(false);
// derived state
const isUrl = importMedia === 'link';
@@ -150,6 +153,13 @@ export function ImportChats(props: { onConversationActivate: (conversationId: DC
</Button>
)}
<Button
variant='soft' endDecorator={<CloudUploadIcon />} sx={{ minWidth: 240, justifyContent: 'space-between' }}
onClick={() => setTypingMindModalOpen(true)}
>
TypingMind · Export
</Button>
{/* Insert to Restore a Flash */}
<FlashRestore unlockRestore={true} />
@@ -193,5 +203,14 @@ export function ImportChats(props: { onConversationActivate: (conversationId: DC
{/* import outcome */}
{!!importOutcome && <ImportOutcomeModal outcome={importOutcome} rawJson={importJson} onClose={handleImportOutcomeClosed} />}
{/* TypingMind import modal */}
<DataImportModal
vendorId='typingmind'
vendorLabel='TypingMind'
open={typingMindModalOpen}
onConversationActivate={props.onConversationActivate}
onClose={() => setTypingMindModalOpen(false)}
/>
</>;
}