mirror of
https://github.com/enricoros/big-AGI.git
synced 2026-05-10 21:50:14 -07:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| d847649fe5 |
@@ -0,0 +1,176 @@
|
||||
/**
|
||||
* Main import orchestration function
|
||||
* Coordinates the full import pipeline: parse -> validate -> transform -> import
|
||||
*/
|
||||
|
||||
import { useChatStore } from '~/common/stores/chat/store-chats';
|
||||
import type { DConversation } from '~/common/stores/chat/chat.conversation';
|
||||
import type { ImportResult, ImportContext } from './data.types';
|
||||
|
||||
import { calculateFileHash, attachLineage } from './data.lineage';
|
||||
import { validateConversations, sanitizeConversation } from './pipeline/import.validator';
|
||||
import { resolveAllConflictsByRename } from './pipeline/import.conflict-resolver';
|
||||
|
||||
|
||||
/**
|
||||
* Import options
|
||||
*/
|
||||
export interface ImportOptions {
|
||||
dryRun?: boolean; // If true, validate but don't actually import
|
||||
preventClash?: boolean; // If true, rename conflicting IDs
|
||||
sanitize?: boolean; // If true, attempt to repair invalid data
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Generic import function for any vendor data
|
||||
*/
|
||||
export async function importVendorData(
|
||||
file: File,
|
||||
parseFile: (file: File) => Promise<any>,
|
||||
transformToConversations: (data: any) => Promise<any>,
|
||||
vendorId: string,
|
||||
options: ImportOptions = {},
|
||||
): Promise<ImportResult> {
|
||||
|
||||
const {
|
||||
dryRun = false,
|
||||
preventClash = true,
|
||||
sanitize = true,
|
||||
} = options;
|
||||
|
||||
// Initialize result
|
||||
const context: ImportContext = {
|
||||
fileName: file.name,
|
||||
fileSize: file.size,
|
||||
fileHash: await calculateFileHash(file),
|
||||
importedAt: Date.now(),
|
||||
vendorId,
|
||||
};
|
||||
|
||||
const result: ImportResult = {
|
||||
success: false,
|
||||
conversations: [],
|
||||
warnings: [],
|
||||
errors: [],
|
||||
stats: {
|
||||
conversationsImported: 0,
|
||||
messagesImported: 0,
|
||||
foldersImported: 0,
|
||||
charactersImported: 0,
|
||||
unsupportedItemsSkipped: 0,
|
||||
},
|
||||
context,
|
||||
};
|
||||
|
||||
try {
|
||||
// Step 1: Parse the file
|
||||
const parseResult = await parseFile(file);
|
||||
|
||||
if (!parseResult.success || !parseResult.data) {
|
||||
result.errors.push({
|
||||
type: 'parse-error',
|
||||
message: parseResult.error || 'Failed to parse file',
|
||||
fatal: true,
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
result.warnings.push(...parseResult.warnings);
|
||||
|
||||
// Step 2: Transform to conversations
|
||||
const transformResult = await transformToConversations(parseResult.data);
|
||||
|
||||
if (!transformResult.conversations || transformResult.conversations.length === 0) {
|
||||
result.errors.push({
|
||||
type: 'transform-error',
|
||||
message: 'No conversations found in file',
|
||||
fatal: true,
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
let conversations = transformResult.conversations;
|
||||
result.warnings.push(...transformResult.warnings);
|
||||
|
||||
if (transformResult.unsupportedFeatures?.length > 0) {
|
||||
result.warnings.push({
|
||||
type: 'unsupported-feature',
|
||||
message: `Unsupported features: ${transformResult.unsupportedFeatures.join(', ')}`,
|
||||
});
|
||||
result.stats.unsupportedItemsSkipped = transformResult.unsupportedFeatures.length;
|
||||
}
|
||||
|
||||
// Step 3: Sanitize if requested
|
||||
if (sanitize) {
|
||||
conversations = conversations.map(sanitizeConversation);
|
||||
}
|
||||
|
||||
// Step 4: Validate
|
||||
const validationResult = validateConversations(conversations);
|
||||
result.warnings.push(...validationResult.warnings);
|
||||
result.errors.push(...validationResult.errors);
|
||||
|
||||
if (!validationResult.valid) {
|
||||
result.errors.push({
|
||||
type: 'validation-error',
|
||||
message: 'Validation failed - see errors above',
|
||||
fatal: true,
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
// Step 5: Resolve conflicts
|
||||
if (preventClash) {
|
||||
const existingConversations = useChatStore.getState().conversations;
|
||||
conversations = resolveAllConflictsByRename(conversations, existingConversations);
|
||||
}
|
||||
|
||||
// Step 6: Attach lineage metadata
|
||||
for (const conversation of conversations) {
|
||||
const originalId = (conversation.metadata as any)?.typingmind?.chatId || conversation.id;
|
||||
attachLineage(conversation, context, originalId);
|
||||
}
|
||||
|
||||
// Step 7: Calculate stats
|
||||
result.stats.conversationsImported = conversations.length;
|
||||
result.stats.messagesImported = conversations.reduce(
|
||||
(sum: number, conv: DConversation) => sum + conv.messages.length,
|
||||
0,
|
||||
);
|
||||
result.stats.charactersImported = conversations.reduce(
|
||||
(sum: number, conv: DConversation) => sum + conv.messages.reduce(
|
||||
(msgSum: number, msg) => msgSum + msg.fragments.reduce(
|
||||
(fragSum: number, frag) => {
|
||||
const text = (frag as any).part?.text;
|
||||
return fragSum + (typeof text === 'string' ? text.length : 0);
|
||||
},
|
||||
0,
|
||||
),
|
||||
0,
|
||||
),
|
||||
0,
|
||||
);
|
||||
|
||||
// Step 8: Import (if not dry run)
|
||||
if (!dryRun) {
|
||||
const chatStore = useChatStore.getState();
|
||||
for (const conversation of conversations) {
|
||||
chatStore.importConversation(conversation, preventClash);
|
||||
}
|
||||
}
|
||||
|
||||
result.conversations = conversations;
|
||||
result.success = true;
|
||||
|
||||
return result;
|
||||
|
||||
} catch (error) {
|
||||
result.errors.push({
|
||||
type: 'parse-error',
|
||||
message: error instanceof Error ? error.message : 'Unknown error',
|
||||
fatal: true,
|
||||
});
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
/**
|
||||
* Data lineage and provenance tracking for imported conversations
|
||||
* Provides import history, re-import detection, and file hash tracking
|
||||
*/
|
||||
|
||||
import type { DConversation } from '~/common/stores/chat/chat.conversation';
|
||||
import type { ImportContext } from './data.types';
|
||||
|
||||
|
||||
// Lineage metadata stored in conversation metadata
|
||||
|
||||
export interface DConversationLineage {
|
||||
// Import provenance
|
||||
importSource: {
|
||||
vendorId: string; // 'typingmind', 'chatgpt', etc.
|
||||
fileName: string; // Original filename
|
||||
fileHash: string; // SHA-256 hash of the file
|
||||
importedAt: number; // Unix timestamp
|
||||
};
|
||||
|
||||
// Re-import tracking
|
||||
reimportCount?: number; // Number of times this was reimported
|
||||
lastReimportAt?: number; // Last reimport timestamp
|
||||
|
||||
// Original IDs (for conflict resolution)
|
||||
originalIds?: {
|
||||
conversationId?: string;
|
||||
chatId?: string;
|
||||
folderId?: string;
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Calculate SHA-256 hash of a file
|
||||
*/
|
||||
export async function calculateFileHash(file: File): Promise<string> {
|
||||
const buffer = await file.arrayBuffer();
|
||||
const hashBuffer = await crypto.subtle.digest('SHA-256', buffer);
|
||||
const hashArray = Array.from(new Uint8Array(hashBuffer));
|
||||
const hashHex = hashArray.map(b => b.toString(16).padStart(2, '0')).join('');
|
||||
return hashHex;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Attach lineage metadata to a conversation
|
||||
*/
|
||||
export function attachLineage(
|
||||
conversation: DConversation,
|
||||
context: ImportContext,
|
||||
originalId?: string,
|
||||
): void {
|
||||
const lineage: DConversationLineage = {
|
||||
importSource: {
|
||||
vendorId: context.vendorId,
|
||||
fileName: context.fileName,
|
||||
fileHash: context.fileHash,
|
||||
importedAt: context.importedAt,
|
||||
},
|
||||
};
|
||||
|
||||
if (originalId) {
|
||||
lineage.originalIds = {
|
||||
conversationId: originalId,
|
||||
};
|
||||
}
|
||||
|
||||
// Store in conversation metadata (extending the DConversation type)
|
||||
if (!(conversation as any).metadata) {
|
||||
(conversation as any).metadata = {};
|
||||
}
|
||||
(conversation as any).metadata.lineage = lineage;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Get lineage metadata from a conversation
|
||||
*/
|
||||
export function getLineage(conversation: DConversation): DConversationLineage | null {
|
||||
return (conversation as any).metadata?.lineage || null;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Check if a conversation was imported from a specific file hash
|
||||
*/
|
||||
export function isFromFile(conversation: DConversation, fileHash: string): boolean {
|
||||
const lineage = getLineage(conversation);
|
||||
return lineage?.importSource.fileHash === fileHash;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Detect if this is a re-import of existing conversations
|
||||
*/
|
||||
export function detectReimport(
|
||||
existingConversations: DConversation[],
|
||||
fileHash: string,
|
||||
): {
|
||||
isReimport: boolean;
|
||||
existingConversations: DConversation[];
|
||||
} {
|
||||
const matches = existingConversations.filter(c => isFromFile(c, fileHash));
|
||||
return {
|
||||
isReimport: matches.length > 0,
|
||||
existingConversations: matches,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Update lineage for a re-import
|
||||
*/
|
||||
export function updateLineageForReimport(conversation: DConversation): void {
|
||||
const lineage = getLineage(conversation);
|
||||
if (!lineage) return;
|
||||
|
||||
lineage.reimportCount = (lineage.reimportCount || 0) + 1;
|
||||
lineage.lastReimportAt = Date.now();
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
/**
|
||||
* Core types for the data import system
|
||||
*/
|
||||
|
||||
import type { DConversation } from '~/common/stores/chat/chat.conversation';
|
||||
|
||||
|
||||
// Import Source
|
||||
|
||||
export type ImportSourceId = string;
|
||||
|
||||
export interface ImportSource {
|
||||
readonly id: ImportSourceId;
|
||||
readonly label: string;
|
||||
readonly description?: string;
|
||||
readonly vendorId: string;
|
||||
}
|
||||
|
||||
|
||||
// Import Context
|
||||
|
||||
export interface ImportContext {
|
||||
fileName: string;
|
||||
fileSize: number;
|
||||
fileHash: string;
|
||||
importedAt: number;
|
||||
vendorId: string;
|
||||
}
|
||||
|
||||
|
||||
// Import Result
|
||||
|
||||
export interface ImportResult {
|
||||
success: boolean;
|
||||
conversations: DConversation[];
|
||||
warnings: ImportWarning[];
|
||||
errors: ImportError[];
|
||||
stats: ImportStats;
|
||||
context: ImportContext;
|
||||
}
|
||||
|
||||
export interface ImportWarning {
|
||||
type: 'unsupported-feature' | 'data-loss' | 'approximation' | 'missing-data';
|
||||
message: string;
|
||||
details?: string;
|
||||
affectedConversationIds?: string[];
|
||||
}
|
||||
|
||||
export interface ImportError {
|
||||
type: 'parse-error' | 'validation-error' | 'transform-error' | 'conflict-error';
|
||||
message: string;
|
||||
details?: string;
|
||||
fatal: boolean;
|
||||
}
|
||||
|
||||
export interface ImportStats {
|
||||
conversationsImported: number;
|
||||
messagesImported: number;
|
||||
foldersImported: number;
|
||||
charactersImported: number;
|
||||
unsupportedItemsSkipped: number;
|
||||
}
|
||||
|
||||
|
||||
// Conflict Resolution
|
||||
|
||||
export type ConflictResolutionStrategy = 'skip' | 'rename' | 'overwrite' | 'merge';
|
||||
|
||||
export interface ConflictInfo {
|
||||
type: 'conversation-id' | 'folder-id';
|
||||
existingId: string;
|
||||
proposedId: string;
|
||||
resolution: ConflictResolutionStrategy;
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
/**
|
||||
* Data import module
|
||||
* Provides a modular system for importing conversations from various sources
|
||||
*/
|
||||
|
||||
// Core types
|
||||
export type {
|
||||
ImportSourceId,
|
||||
ImportSource,
|
||||
ImportContext,
|
||||
ImportResult,
|
||||
ImportWarning,
|
||||
ImportError,
|
||||
ImportStats,
|
||||
ConflictResolutionStrategy,
|
||||
ConflictInfo,
|
||||
} from './data.types';
|
||||
|
||||
// Lineage tracking
|
||||
export {
|
||||
calculateFileHash,
|
||||
attachLineage,
|
||||
getLineage,
|
||||
isFromFile,
|
||||
detectReimport,
|
||||
updateLineageForReimport,
|
||||
} from './data.lineage';
|
||||
|
||||
export type { DConversationLineage } from './data.lineage';
|
||||
|
||||
// Import orchestration
|
||||
export { importVendorData } from './data.import';
|
||||
export type { ImportOptions } from './data.import';
|
||||
|
||||
// Vendor types
|
||||
export type {
|
||||
VendorId,
|
||||
IDataVendor,
|
||||
ParseResult,
|
||||
TransformResult,
|
||||
ValidationResult,
|
||||
} from './vendors/vendor.types';
|
||||
|
||||
// Vendor registry
|
||||
export {
|
||||
registerDataVendor,
|
||||
getDataVendor,
|
||||
getAllDataVendors,
|
||||
hasDataVendor,
|
||||
} from './vendors/vendor.registry';
|
||||
|
||||
// UI components
|
||||
export { DataImportModal } from './ui/DataImportModal';
|
||||
export { ImportConfirmStep } from './ui/ImportConfirmStep';
|
||||
export { ImportResultModal } from './ui/ImportResultModal';
|
||||
|
||||
// TypingMind vendor
|
||||
export { importTypingMindData } from './vendors/typingmind/typingmind.import-function';
|
||||
export { TypingMindImporter } from './vendors/typingmind/typingmind.importer';
|
||||
@@ -0,0 +1,143 @@
|
||||
/**
|
||||
* Conflict resolution for imports
|
||||
* Handles ID conflicts and provides resolution strategies
|
||||
*/
|
||||
|
||||
import { agiUuid } from '~/common/util/idUtils';
|
||||
import type { DConversation } from '~/common/stores/chat/chat.conversation';
|
||||
import type { ConflictInfo, ConflictResolutionStrategy } from '../data.types';
|
||||
|
||||
|
||||
/**
|
||||
* Detect conflicts between imported and existing conversations
|
||||
*/
|
||||
export function detectConflicts(
|
||||
importedConversations: DConversation[],
|
||||
existingConversations: DConversation[],
|
||||
): ConflictInfo[] {
|
||||
const conflicts: ConflictInfo[] = [];
|
||||
const existingIds = new Set(existingConversations.map(c => c.id));
|
||||
|
||||
for (const imported of importedConversations) {
|
||||
if (existingIds.has(imported.id)) {
|
||||
conflicts.push({
|
||||
type: 'conversation-id',
|
||||
existingId: imported.id,
|
||||
proposedId: imported.id,
|
||||
resolution: 'rename', // Default strategy
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return conflicts;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Resolve conflicts using the specified strategy
|
||||
*/
|
||||
export function resolveConflicts(
|
||||
importedConversations: DConversation[],
|
||||
conflicts: ConflictInfo[],
|
||||
strategy: ConflictResolutionStrategy = 'rename',
|
||||
): DConversation[] {
|
||||
if (conflicts.length === 0) {
|
||||
return importedConversations;
|
||||
}
|
||||
|
||||
const conflictMap = new Map(conflicts.map(c => [c.existingId, c]));
|
||||
|
||||
return importedConversations.map(conversation => {
|
||||
const conflict = conflictMap.get(conversation.id);
|
||||
|
||||
if (!conflict) {
|
||||
return conversation;
|
||||
}
|
||||
|
||||
switch (strategy) {
|
||||
case 'rename':
|
||||
return {
|
||||
...conversation,
|
||||
id: agiUuid('chat-dconversation'),
|
||||
autoTitle: conversation.autoTitle
|
||||
? `${conversation.autoTitle} (imported)`
|
||||
: conversation.userTitle
|
||||
? `${conversation.userTitle} (imported)`
|
||||
: 'Imported conversation',
|
||||
};
|
||||
|
||||
case 'skip':
|
||||
// Mark for filtering later
|
||||
return null as any;
|
||||
|
||||
case 'overwrite':
|
||||
// Keep the original ID, will overwrite
|
||||
return conversation;
|
||||
|
||||
case 'merge':
|
||||
// For now, same as rename
|
||||
// TODO: Implement actual merge logic
|
||||
return {
|
||||
...conversation,
|
||||
id: agiUuid('chat-dconversation'),
|
||||
};
|
||||
|
||||
default:
|
||||
return conversation;
|
||||
}
|
||||
}).filter(Boolean);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Check if a conversation ID already exists
|
||||
*/
|
||||
export function hasConflict(
|
||||
conversationId: string,
|
||||
existingConversations: DConversation[],
|
||||
): boolean {
|
||||
return existingConversations.some(c => c.id === conversationId);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Generate a safe ID that doesn't conflict
|
||||
*/
|
||||
export function generateSafeId(
|
||||
existingConversations: DConversation[],
|
||||
): string {
|
||||
const existingIds = new Set(existingConversations.map(c => c.id));
|
||||
let newId: string;
|
||||
let attempts = 0;
|
||||
|
||||
do {
|
||||
newId = agiUuid('chat-dconversation');
|
||||
attempts++;
|
||||
} while (existingIds.has(newId) && attempts < 100);
|
||||
|
||||
return newId;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Resolve all conversation ID conflicts by renaming
|
||||
*/
|
||||
export function resolveAllConflictsByRename(
|
||||
importedConversations: DConversation[],
|
||||
existingConversations: DConversation[],
|
||||
): DConversation[] {
|
||||
const existingIds = new Set(existingConversations.map(c => c.id));
|
||||
|
||||
return importedConversations.map(conversation => {
|
||||
if (existingIds.has(conversation.id)) {
|
||||
return {
|
||||
...conversation,
|
||||
id: generateSafeId(existingConversations),
|
||||
autoTitle: conversation.autoTitle
|
||||
? `${conversation.autoTitle} (imported)`
|
||||
: 'Imported conversation',
|
||||
};
|
||||
}
|
||||
return conversation;
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
/**
|
||||
* Transform utilities for converting vendor data to Big-AGI format
|
||||
*/
|
||||
|
||||
import { agiUuid } from '~/common/util/idUtils';
|
||||
import { createTextContentFragment } from '~/common/stores/chat/chat.fragments';
|
||||
import type { DMessage, DMessageRole } from '~/common/stores/chat/chat.message';
|
||||
import type { DConversation } from '~/common/stores/chat/chat.conversation';
|
||||
import { defaultSystemPurposeId } from '../../../data';
|
||||
|
||||
|
||||
/**
|
||||
* Convert ISO timestamp string to Unix milliseconds
|
||||
*/
|
||||
export function isoToUnixMs(isoString: string | number): number {
|
||||
if (typeof isoString === 'number') {
|
||||
// Already a number - check if it's seconds or milliseconds
|
||||
return isoString < 10000000000 ? isoString * 1000 : isoString;
|
||||
}
|
||||
|
||||
try {
|
||||
const timestamp = new Date(isoString).getTime();
|
||||
return Number.isFinite(timestamp) ? timestamp : Date.now();
|
||||
} catch {
|
||||
return Date.now();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Normalize message content to text string
|
||||
* Handles both string and array formats
|
||||
*/
|
||||
export function normalizeMessageContent(content: unknown): string {
|
||||
// Already a string
|
||||
if (typeof content === 'string') {
|
||||
return content;
|
||||
}
|
||||
|
||||
// Array of content parts
|
||||
if (Array.isArray(content)) {
|
||||
return content
|
||||
.map(part => {
|
||||
if (typeof part === 'string') return part;
|
||||
if (part && typeof part === 'object' && 'text' in part) {
|
||||
return String(part.text);
|
||||
}
|
||||
if (part && typeof part === 'object' && 'type' in part && part.type === 'text' && 'text' in part) {
|
||||
return String(part.text);
|
||||
}
|
||||
return '';
|
||||
})
|
||||
.filter(text => text.length > 0)
|
||||
.join('\n\n');
|
||||
}
|
||||
|
||||
// Unknown format
|
||||
return '';
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Create a DMessage from basic components
|
||||
*/
|
||||
export function createImportedMessage(
|
||||
role: DMessageRole,
|
||||
content: string,
|
||||
originalId?: string,
|
||||
createdAt?: string | number,
|
||||
): DMessage {
|
||||
const message: DMessage = {
|
||||
id: originalId || agiUuid('chat-dmessage'),
|
||||
role,
|
||||
fragments: [createTextContentFragment(content)],
|
||||
tokenCount: 0,
|
||||
created: createdAt ? isoToUnixMs(createdAt) : Date.now(),
|
||||
updated: null,
|
||||
};
|
||||
|
||||
return message;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Create an empty DConversation with defaults
|
||||
*/
|
||||
export function createImportedConversation(
|
||||
title?: string,
|
||||
originalId?: string,
|
||||
createdAt?: string | number,
|
||||
updatedAt?: string | number,
|
||||
): DConversation {
|
||||
const now = Date.now();
|
||||
|
||||
const conversation: DConversation = {
|
||||
id: originalId || agiUuid('chat-dconversation'),
|
||||
messages: [],
|
||||
systemPurposeId: defaultSystemPurposeId,
|
||||
tokenCount: 0,
|
||||
created: createdAt ? isoToUnixMs(createdAt) : now,
|
||||
updated: updatedAt ? isoToUnixMs(updatedAt) : now,
|
||||
_abortController: null,
|
||||
};
|
||||
|
||||
if (title) {
|
||||
conversation.autoTitle = title;
|
||||
}
|
||||
|
||||
return conversation;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Generate a unique ID that won't conflict with existing IDs
|
||||
*/
|
||||
export function generateNonConflictingId(
|
||||
existingIds: Set<string>,
|
||||
prefix: string = 'imported',
|
||||
): string {
|
||||
let attempt = 0;
|
||||
let newId: string;
|
||||
|
||||
do {
|
||||
newId = attempt === 0
|
||||
? agiUuid('chat-dconversation')
|
||||
: `${prefix}-${Date.now()}-${attempt}`;
|
||||
attempt++;
|
||||
} while (existingIds.has(newId) && attempt < 100);
|
||||
|
||||
if (attempt >= 100) {
|
||||
// Fallback: use timestamp + random
|
||||
newId = `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 11)}`;
|
||||
}
|
||||
|
||||
return newId;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Deduplicate messages by ID, keeping first occurrence
|
||||
*/
|
||||
export function deduplicateMessages(messages: DMessage[]): DMessage[] {
|
||||
const seen = new Set<string>();
|
||||
return messages.filter(msg => {
|
||||
if (seen.has(msg.id)) {
|
||||
return false;
|
||||
}
|
||||
seen.add(msg.id);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Sort messages by creation timestamp
|
||||
*/
|
||||
export function sortMessagesByTimestamp(messages: DMessage[]): DMessage[] {
|
||||
return [...messages].sort((a, b) => a.created - b.created);
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
/**
|
||||
* Validation utilities for imported data
|
||||
*/
|
||||
|
||||
import type { DConversation } from '~/common/stores/chat/chat.conversation';
|
||||
import type { DMessage } from '~/common/stores/chat/chat.message';
|
||||
import type { ImportError, ImportWarning } from '../data.types';
|
||||
|
||||
|
||||
export interface ValidationResult {
|
||||
valid: boolean;
|
||||
errors: ImportError[];
|
||||
warnings: ImportWarning[];
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Validate a conversation structure
|
||||
*/
|
||||
export function validateConversation(conversation: DConversation): ValidationResult {
|
||||
const errors: ImportError[] = [];
|
||||
const warnings: ImportWarning[] = [];
|
||||
|
||||
// Required fields
|
||||
if (!conversation.id) {
|
||||
errors.push({
|
||||
type: 'validation-error',
|
||||
message: 'Conversation missing required ID',
|
||||
fatal: true,
|
||||
});
|
||||
}
|
||||
|
||||
if (!conversation.messages) {
|
||||
errors.push({
|
||||
type: 'validation-error',
|
||||
message: 'Conversation missing messages array',
|
||||
fatal: true,
|
||||
});
|
||||
}
|
||||
|
||||
// Validate timestamps
|
||||
if (conversation.created && (conversation.created < 0 || !Number.isFinite(conversation.created))) {
|
||||
warnings.push({
|
||||
type: 'data-loss',
|
||||
message: 'Invalid created timestamp, using current time',
|
||||
affectedConversationIds: [conversation.id],
|
||||
});
|
||||
}
|
||||
|
||||
// Validate messages
|
||||
if (conversation.messages && Array.isArray(conversation.messages)) {
|
||||
for (let i = 0; i < conversation.messages.length; i++) {
|
||||
const messageResult = validateMessage(conversation.messages[i], i);
|
||||
errors.push(...messageResult.errors);
|
||||
warnings.push(...messageResult.warnings);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
valid: errors.filter(e => e.fatal).length === 0,
|
||||
errors,
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Validate a message structure
|
||||
*/
|
||||
export function validateMessage(message: DMessage, index: number): ValidationResult {
|
||||
const errors: ImportError[] = [];
|
||||
const warnings: ImportWarning[] = [];
|
||||
|
||||
// Required fields
|
||||
if (!message.id) {
|
||||
errors.push({
|
||||
type: 'validation-error',
|
||||
message: `Message at index ${index} missing required ID`,
|
||||
fatal: true,
|
||||
});
|
||||
}
|
||||
|
||||
if (!message.role || !['user', 'assistant', 'system'].includes(message.role)) {
|
||||
errors.push({
|
||||
type: 'validation-error',
|
||||
message: `Message at index ${index} has invalid role: ${message.role}`,
|
||||
fatal: true,
|
||||
});
|
||||
}
|
||||
|
||||
if (!message.fragments || !Array.isArray(message.fragments)) {
|
||||
errors.push({
|
||||
type: 'validation-error',
|
||||
message: `Message at index ${index} missing fragments array`,
|
||||
fatal: true,
|
||||
});
|
||||
}
|
||||
|
||||
// Validate timestamps
|
||||
if (message.created && (message.created < 0 || !Number.isFinite(message.created))) {
|
||||
warnings.push({
|
||||
type: 'data-loss',
|
||||
message: `Message at index ${index} has invalid created timestamp`,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
valid: errors.filter(e => e.fatal).length === 0,
|
||||
errors,
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Validate all conversations in a batch
|
||||
*/
|
||||
export function validateConversations(conversations: DConversation[]): ValidationResult {
|
||||
const allErrors: ImportError[] = [];
|
||||
const allWarnings: ImportWarning[] = [];
|
||||
|
||||
for (const conversation of conversations) {
|
||||
const result = validateConversation(conversation);
|
||||
allErrors.push(...result.errors);
|
||||
allWarnings.push(...result.warnings);
|
||||
}
|
||||
|
||||
return {
|
||||
valid: allErrors.filter(e => e.fatal).length === 0,
|
||||
errors: allErrors,
|
||||
warnings: allWarnings,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Sanitize and repair a conversation to make it valid
|
||||
*/
|
||||
export function sanitizeConversation(conversation: DConversation): DConversation {
|
||||
// Ensure required fields
|
||||
if (!conversation.id) {
|
||||
conversation.id = `imported-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`;
|
||||
}
|
||||
|
||||
if (!conversation.messages) {
|
||||
conversation.messages = [];
|
||||
}
|
||||
|
||||
if (!conversation.created || conversation.created < 0) {
|
||||
conversation.created = Date.now();
|
||||
}
|
||||
|
||||
if (!conversation.updated) {
|
||||
conversation.updated = conversation.created;
|
||||
}
|
||||
|
||||
if (typeof conversation.tokenCount !== 'number') {
|
||||
conversation.tokenCount = 0;
|
||||
}
|
||||
|
||||
// Sanitize messages
|
||||
conversation.messages = conversation.messages
|
||||
.map((msg, index) => sanitizeMessage(msg, index))
|
||||
.filter(msg => msg !== null) as DMessage[];
|
||||
|
||||
return conversation;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Sanitize and repair a message to make it valid
|
||||
*/
|
||||
function sanitizeMessage(message: DMessage, index: number): DMessage | null {
|
||||
// Skip messages with fatal errors
|
||||
if (!message || typeof message !== 'object') {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Ensure required fields
|
||||
if (!message.id) {
|
||||
message.id = `imported-msg-${Date.now()}-${index}`;
|
||||
}
|
||||
|
||||
if (!message.role || !['user', 'assistant', 'system'].includes(message.role)) {
|
||||
message.role = 'user'; // Default to user
|
||||
}
|
||||
|
||||
if (!message.fragments || !Array.isArray(message.fragments)) {
|
||||
message.fragments = [];
|
||||
}
|
||||
|
||||
if (!message.created || message.created < 0) {
|
||||
message.created = Date.now();
|
||||
}
|
||||
|
||||
if (typeof message.tokenCount !== 'number') {
|
||||
message.tokenCount = 0;
|
||||
}
|
||||
|
||||
return message;
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
/**
|
||||
* tRPC router for data import operations
|
||||
* Provides server-side endpoints for data import if needed
|
||||
*/
|
||||
|
||||
import * as z from 'zod/v4';
|
||||
import { createTRPCRouter, publicProcedure } from '~/server/trpc/trpc.server';
|
||||
|
||||
|
||||
/**
|
||||
* Data import router
|
||||
* Currently minimal - most import operations happen client-side
|
||||
*/
|
||||
export const dataRouter = createTRPCRouter({
|
||||
|
||||
/**
|
||||
* Validate a data import file (server-side validation)
|
||||
* This can be used for large files or security checks
|
||||
*/
|
||||
validateImportFile: publicProcedure
|
||||
.input(z.object({
|
||||
vendorId: z.enum(['typingmind', 'chatgpt', 'bigagi']),
|
||||
fileSize: z.number(),
|
||||
fileHash: z.string(),
|
||||
}))
|
||||
.output(z.object({
|
||||
valid: z.boolean(),
|
||||
message: z.string().optional(),
|
||||
}))
|
||||
.query(async ({ input }: { input: { vendorId: string; fileSize: number; fileHash: string } }) => {
|
||||
// Basic validation
|
||||
const MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
|
||||
|
||||
if (input.fileSize > MAX_FILE_SIZE) {
|
||||
return {
|
||||
valid: false,
|
||||
message: `File too large: ${(input.fileSize / 1024 / 1024).toFixed(2)}MB (max ${MAX_FILE_SIZE / 1024 / 1024}MB)`,
|
||||
};
|
||||
}
|
||||
|
||||
if (input.fileSize === 0) {
|
||||
return {
|
||||
valid: false,
|
||||
message: 'File is empty',
|
||||
};
|
||||
}
|
||||
|
||||
// All checks passed
|
||||
return {
|
||||
valid: true,
|
||||
};
|
||||
}),
|
||||
|
||||
/**
|
||||
* Get import statistics
|
||||
* Returns information about previous imports if tracked
|
||||
*/
|
||||
getImportStats: publicProcedure
|
||||
.output(z.object({
|
||||
totalImports: z.number(),
|
||||
recentImports: z.array(z.object({
|
||||
vendorId: z.string(),
|
||||
fileName: z.string(),
|
||||
conversationCount: z.number(),
|
||||
importedAt: z.number(),
|
||||
})),
|
||||
}))
|
||||
.query(async () => {
|
||||
// For now, return empty stats
|
||||
// This could be expanded to track import history in the future
|
||||
return {
|
||||
totalImports: 0,
|
||||
recentImports: [],
|
||||
};
|
||||
}),
|
||||
|
||||
});
|
||||
|
||||
|
||||
export type DataRouter = typeof dataRouter;
|
||||
@@ -0,0 +1,226 @@
|
||||
/**
|
||||
* Main data import modal component
|
||||
* Provides a multi-step import flow: file selection -> validation -> confirmation -> import
|
||||
*/
|
||||
|
||||
import * as React from 'react';
|
||||
import { Box, Button, CircularProgress, Typography } from '@mui/joy';
|
||||
import FileUploadIcon from '@mui/icons-material/FileUpload';
|
||||
|
||||
import { GoodModal } from '~/common/components/modals/GoodModal';
|
||||
import type { DConversationId } from '~/common/stores/chat/chat.conversation';
|
||||
import type { VendorId } from '../vendors/vendor.types';
|
||||
import type { ImportResult } from '../data.types';
|
||||
|
||||
import { ImportConfirmStep } from './ImportConfirmStep';
|
||||
import { ImportResultModal } from './ImportResultModal';
|
||||
|
||||
|
||||
interface DataImportModalProps {
|
||||
vendorId: VendorId;
|
||||
vendorLabel: string;
|
||||
open: boolean;
|
||||
onConversationActivate?: (conversationId: DConversationId) => void;
|
||||
onClose: () => void;
|
||||
}
|
||||
|
||||
|
||||
type ImportStep = 'select' | 'processing' | 'confirm' | 'importing' | 'complete';
|
||||
|
||||
|
||||
export function DataImportModal(props: DataImportModalProps) {
|
||||
const { vendorId, vendorLabel, open, onClose } = props;
|
||||
|
||||
// State
|
||||
const [step, setStep] = React.useState<ImportStep>('select');
|
||||
const [selectedFile, setSelectedFile] = React.useState<File | null>(null);
|
||||
const [importPreview, setImportPreview] = React.useState<any>(null);
|
||||
const [importResult, setImportResult] = React.useState<ImportResult | null>(null);
|
||||
const [error, setError] = React.useState<string | null>(null);
|
||||
|
||||
// File input ref
|
||||
const fileInputRef = React.useRef<HTMLInputElement>(null);
|
||||
|
||||
|
||||
const handleFileSelect = async (event: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const file = event.target.files?.[0];
|
||||
if (!file) return;
|
||||
|
||||
setSelectedFile(file);
|
||||
setStep('processing');
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
// Dynamic import of the vendor importer
|
||||
const { importTypingMindData } = await import('../vendors/typingmind/typingmind.import-function');
|
||||
|
||||
// TODO: Use vendor registry to get the correct importer
|
||||
// For now, only TypingMind is supported
|
||||
if (vendorId !== 'typingmind') {
|
||||
throw new Error(`Vendor ${vendorId} not yet supported`);
|
||||
}
|
||||
|
||||
// Parse and validate the file
|
||||
const result = await importTypingMindData(file, { dryRun: true });
|
||||
|
||||
if (!result.success) {
|
||||
setError(result.errors[0]?.message || 'Failed to parse file');
|
||||
setStep('select');
|
||||
return;
|
||||
}
|
||||
|
||||
setImportPreview(result);
|
||||
setStep('confirm');
|
||||
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Unknown error');
|
||||
setStep('select');
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
const handleConfirmImport = async () => {
|
||||
if (!selectedFile) return;
|
||||
|
||||
setStep('importing');
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
// Dynamic import of the vendor importer
|
||||
const { importTypingMindData } = await import('../vendors/typingmind/typingmind.import-function');
|
||||
|
||||
// Actually perform the import
|
||||
const result = await importTypingMindData(selectedFile, { dryRun: false });
|
||||
|
||||
setImportResult(result);
|
||||
setStep('complete');
|
||||
|
||||
// Activate the last imported conversation
|
||||
if (result.success && result.conversations.length > 0 && props.onConversationActivate) {
|
||||
const lastConv = result.conversations[result.conversations.length - 1];
|
||||
props.onConversationActivate(lastConv.id);
|
||||
}
|
||||
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Unknown error');
|
||||
setStep('confirm');
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
const handleCancelImport = () => {
|
||||
setStep('select');
|
||||
setSelectedFile(null);
|
||||
setImportPreview(null);
|
||||
setError(null);
|
||||
};
|
||||
|
||||
|
||||
const handleCloseComplete = () => {
|
||||
setStep('select');
|
||||
setSelectedFile(null);
|
||||
setImportPreview(null);
|
||||
setImportResult(null);
|
||||
setError(null);
|
||||
onClose();
|
||||
};
|
||||
|
||||
|
||||
const handleOpenFilePicker = () => {
|
||||
fileInputRef.current?.click();
|
||||
};
|
||||
|
||||
|
||||
// Render step content
|
||||
const renderStepContent = () => {
|
||||
switch (step) {
|
||||
case 'select':
|
||||
return (
|
||||
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2, alignItems: 'center', py: 4 }}>
|
||||
<Typography level='body-md'>
|
||||
Select a {vendorLabel} export file to import
|
||||
</Typography>
|
||||
|
||||
<input
|
||||
ref={fileInputRef}
|
||||
type='file'
|
||||
accept='.json,application/json'
|
||||
onChange={handleFileSelect}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
|
||||
<Button
|
||||
variant='solid'
|
||||
color='primary'
|
||||
size='lg'
|
||||
startDecorator={<FileUploadIcon />}
|
||||
onClick={handleOpenFilePicker}
|
||||
>
|
||||
Choose File
|
||||
</Button>
|
||||
|
||||
{error && (
|
||||
<Typography level='body-sm' color='danger'>
|
||||
{error}
|
||||
</Typography>
|
||||
)}
|
||||
</Box>
|
||||
);
|
||||
|
||||
case 'processing':
|
||||
return (
|
||||
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2, alignItems: 'center', py: 4 }}>
|
||||
<CircularProgress />
|
||||
<Typography level='body-md'>Processing file...</Typography>
|
||||
</Box>
|
||||
);
|
||||
|
||||
case 'confirm':
|
||||
return importPreview ? (
|
||||
<ImportConfirmStep
|
||||
preview={importPreview}
|
||||
vendorLabel={vendorLabel}
|
||||
onConfirm={handleConfirmImport}
|
||||
onCancel={handleCancelImport}
|
||||
/>
|
||||
) : null;
|
||||
|
||||
case 'importing':
|
||||
return (
|
||||
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2, alignItems: 'center', py: 4 }}>
|
||||
<CircularProgress />
|
||||
<Typography level='body-md'>Importing conversations...</Typography>
|
||||
</Box>
|
||||
);
|
||||
|
||||
case 'complete':
|
||||
return null; // Handled by separate modal
|
||||
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
// Show result modal when complete
|
||||
if (step === 'complete' && importResult) {
|
||||
return (
|
||||
<ImportResultModal
|
||||
result={importResult}
|
||||
vendorLabel={vendorLabel}
|
||||
onClose={handleCloseComplete}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
return (
|
||||
<GoodModal
|
||||
open={open && step !== 'complete'}
|
||||
title={`Import from ${vendorLabel}`}
|
||||
onClose={step === 'processing' || step === 'importing' ? undefined : onClose}
|
||||
>
|
||||
<Box>{renderStepContent()}</Box>
|
||||
</GoodModal>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
/**
|
||||
* Confirmation step for data import
|
||||
* Shows preview of what will be imported with warnings
|
||||
*/
|
||||
|
||||
import * as React from 'react';
|
||||
import { Alert, Box, Button, Divider, List, ListItem, Typography } from '@mui/joy';
|
||||
import WarningIcon from '@mui/icons-material/Warning';
|
||||
import InfoIcon from '@mui/icons-material/Info';
|
||||
|
||||
|
||||
interface ImportConfirmStepProps {
|
||||
preview: any;
|
||||
vendorLabel: string;
|
||||
onConfirm: () => void;
|
||||
onCancel: () => void;
|
||||
}
|
||||
|
||||
|
||||
export function ImportConfirmStep(props: ImportConfirmStepProps) {
|
||||
const { preview, vendorLabel, onConfirm, onCancel } = props;
|
||||
|
||||
const conversationCount = preview.stats?.conversationsImported || 0;
|
||||
const messageCount = preview.stats?.messagesImported || 0;
|
||||
const hasWarnings = preview.warnings?.length > 0;
|
||||
const hasErrors = preview.errors?.length > 0;
|
||||
|
||||
return (
|
||||
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
|
||||
|
||||
{/* Summary */}
|
||||
<Alert variant='soft' color='primary' startDecorator={<InfoIcon />}>
|
||||
<Box>
|
||||
<Typography level='title-sm'>Ready to Import</Typography>
|
||||
<Typography level='body-sm'>
|
||||
{conversationCount} conversation{conversationCount !== 1 ? 's' : ''} with {messageCount} message{messageCount !== 1 ? 's' : ''}
|
||||
</Typography>
|
||||
</Box>
|
||||
</Alert>
|
||||
|
||||
{/* Warnings */}
|
||||
{hasWarnings && (
|
||||
<>
|
||||
<Alert variant='soft' color='warning' startDecorator={<WarningIcon />}>
|
||||
<Typography level='title-sm'>Warnings</Typography>
|
||||
</Alert>
|
||||
<List size='sm'>
|
||||
{preview.warnings.slice(0, 5).map((warning: any, idx: number) => (
|
||||
<ListItem key={idx}>
|
||||
<Typography level='body-sm'>
|
||||
{warning.message}
|
||||
</Typography>
|
||||
</ListItem>
|
||||
))}
|
||||
{preview.warnings.length > 5 && (
|
||||
<ListItem>
|
||||
<Typography level='body-sm' fontStyle='italic'>
|
||||
... and {preview.warnings.length - 5} more warning{preview.warnings.length - 5 !== 1 ? 's' : ''}
|
||||
</Typography>
|
||||
</ListItem>
|
||||
)}
|
||||
</List>
|
||||
</>
|
||||
)}
|
||||
|
||||
{/* Errors */}
|
||||
{hasErrors && (
|
||||
<>
|
||||
<Alert variant='soft' color='danger'>
|
||||
<Typography level='title-sm'>Errors</Typography>
|
||||
</Alert>
|
||||
<List size='sm'>
|
||||
{preview.errors.map((error: any, idx: number) => (
|
||||
<ListItem key={idx}>
|
||||
<Typography level='body-sm' color='danger'>
|
||||
{error.message}
|
||||
</Typography>
|
||||
</ListItem>
|
||||
))}
|
||||
</List>
|
||||
</>
|
||||
)}
|
||||
|
||||
<Divider />
|
||||
|
||||
{/* Actions */}
|
||||
<Box sx={{ display: 'flex', gap: 1, justifyContent: 'flex-end' }}>
|
||||
<Button
|
||||
variant='plain'
|
||||
color='neutral'
|
||||
onClick={onCancel}
|
||||
>
|
||||
Cancel
|
||||
</Button>
|
||||
<Button
|
||||
variant='solid'
|
||||
color='primary'
|
||||
onClick={onConfirm}
|
||||
disabled={hasErrors}
|
||||
>
|
||||
Import {conversationCount} Conversation{conversationCount !== 1 ? 's' : ''}
|
||||
</Button>
|
||||
</Box>
|
||||
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
/**
|
||||
* Display results after import completion
|
||||
*/
|
||||
|
||||
import * as React from 'react';
|
||||
import { Alert, Box, Divider, List, ListItem, Typography } from '@mui/joy';
|
||||
import CheckCircleIcon from '@mui/icons-material/CheckCircle';
|
||||
import WarningIcon from '@mui/icons-material/Warning';
|
||||
import ErrorIcon from '@mui/icons-material/Error';
|
||||
|
||||
import { GoodModal } from '~/common/components/modals/GoodModal';
|
||||
import type { ImportResult } from '../data.types';
|
||||
|
||||
|
||||
interface ImportResultModalProps {
|
||||
result: ImportResult;
|
||||
vendorLabel: string;
|
||||
onClose: () => void;
|
||||
}
|
||||
|
||||
|
||||
export function ImportResultModal(props: ImportResultModalProps) {
|
||||
const { result, vendorLabel, onClose } = props;
|
||||
|
||||
const { success, stats, warnings, errors } = result;
|
||||
const hasWarnings = warnings.length > 0;
|
||||
const hasErrors = errors.length > 0;
|
||||
|
||||
return (
|
||||
<GoodModal
|
||||
open
|
||||
title={success ? 'Import Successful' : 'Import Failed'}
|
||||
strongerTitle
|
||||
onClose={onClose}
|
||||
>
|
||||
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
|
||||
|
||||
<Divider />
|
||||
|
||||
{/* Success summary */}
|
||||
{success && (
|
||||
<Alert variant='soft' color='success' startDecorator={<CheckCircleIcon />}>
|
||||
<Box>
|
||||
<Typography level='title-sm'>Import Complete</Typography>
|
||||
<Typography level='body-sm'>
|
||||
Imported {stats.conversationsImported} conversation{stats.conversationsImported !== 1 ? 's' : ''} with {stats.messagesImported} message{stats.messagesImported !== 1 ? 's' : ''}
|
||||
</Typography>
|
||||
</Box>
|
||||
</Alert>
|
||||
)}
|
||||
|
||||
{/* Error summary */}
|
||||
{!success && (
|
||||
<Alert variant='soft' color='danger' startDecorator={<ErrorIcon />}>
|
||||
<Box>
|
||||
<Typography level='title-sm'>Import Failed</Typography>
|
||||
<Typography level='body-sm'>
|
||||
{errors[0]?.message || 'Unknown error occurred'}
|
||||
</Typography>
|
||||
</Box>
|
||||
</Alert>
|
||||
)}
|
||||
|
||||
{/* Statistics */}
|
||||
{success && (
|
||||
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 1 }}>
|
||||
<Typography level='title-sm'>Import Statistics</Typography>
|
||||
<List size='sm'>
|
||||
<ListItem>Conversations: {stats.conversationsImported}</ListItem>
|
||||
<ListItem>Messages: {stats.messagesImported}</ListItem>
|
||||
{stats.charactersImported > 0 && (
|
||||
<ListItem>Characters: {stats.charactersImported.toLocaleString()}</ListItem>
|
||||
)}
|
||||
{stats.unsupportedItemsSkipped > 0 && (
|
||||
<ListItem>Unsupported items skipped: {stats.unsupportedItemsSkipped}</ListItem>
|
||||
)}
|
||||
</List>
|
||||
</Box>
|
||||
)}
|
||||
|
||||
{/* Warnings */}
|
||||
{hasWarnings && (
|
||||
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 1 }}>
|
||||
<Alert variant='soft' color='warning' startDecorator={<WarningIcon />}>
|
||||
<Typography level='title-sm'>
|
||||
{warnings.length} Warning{warnings.length !== 1 ? 's' : ''}
|
||||
</Typography>
|
||||
</Alert>
|
||||
<List size='sm'>
|
||||
{warnings.slice(0, 10).map((warning, idx) => (
|
||||
<ListItem key={idx}>
|
||||
<Typography level='body-sm'>
|
||||
[{warning.type}] {warning.message}
|
||||
</Typography>
|
||||
</ListItem>
|
||||
))}
|
||||
{warnings.length > 10 && (
|
||||
<ListItem>
|
||||
<Typography level='body-sm' fontStyle='italic'>
|
||||
... and {warnings.length - 10} more warning{warnings.length - 10 !== 1 ? 's' : ''}
|
||||
</Typography>
|
||||
</ListItem>
|
||||
)}
|
||||
</List>
|
||||
</Box>
|
||||
)}
|
||||
|
||||
{/* Errors (non-fatal) */}
|
||||
{hasErrors && success && (
|
||||
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 1 }}>
|
||||
<Alert variant='soft' color='danger' startDecorator={<ErrorIcon />}>
|
||||
<Typography level='title-sm'>
|
||||
{errors.length} Error{errors.length !== 1 ? 's' : ''}
|
||||
</Typography>
|
||||
</Alert>
|
||||
<List size='sm'>
|
||||
{errors.slice(0, 5).map((error, idx) => (
|
||||
<ListItem key={idx}>
|
||||
<Typography level='body-sm' color='danger'>
|
||||
[{error.type}] {error.message}
|
||||
</Typography>
|
||||
</ListItem>
|
||||
))}
|
||||
{errors.length > 5 && (
|
||||
<ListItem>
|
||||
<Typography level='body-sm' fontStyle='italic'>
|
||||
... and {errors.length - 5} more error{errors.length - 5 !== 1 ? 's' : ''}
|
||||
</Typography>
|
||||
</ListItem>
|
||||
)}
|
||||
</List>
|
||||
</Box>
|
||||
)}
|
||||
|
||||
{/* Information */}
|
||||
<Typography level='body-sm' sx={{ mt: 2 }}>
|
||||
{success
|
||||
? `The imported conversations are now available in your chat list. ${stats.conversationsImported > 1 ? 'The most recent conversation is' : 'It is'} now active.`
|
||||
: 'Please check the error messages above and try again with a valid export file.'
|
||||
}
|
||||
</Typography>
|
||||
|
||||
</Box>
|
||||
</GoodModal>
|
||||
);
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
/**
|
||||
* TypingMind vendor exports
|
||||
*/
|
||||
|
||||
export { TypingMindImporter } from './typingmind.importer';
|
||||
export { importTypingMindData } from './typingmind.import-function';
|
||||
export { parseTypingMindFile, validateTypingMindFile } from './typingmind.parser';
|
||||
export { transformTypingMindToConversations, getTransformStats } from './typingmind.transformer';
|
||||
export { validateTypingMindSource } from './typingmind.validator';
|
||||
|
||||
export type {
|
||||
TypingMindExport,
|
||||
TypingMindChat,
|
||||
TypingMindMessage,
|
||||
TypingMindFolder,
|
||||
TypingMindUserPrompt,
|
||||
TypingMindUserCharacter,
|
||||
} from './typingmind.schema';
|
||||
@@ -0,0 +1,27 @@
|
||||
/**
|
||||
* High-level import function for TypingMind data
|
||||
* This is the main entry point for TypingMind imports
|
||||
*/
|
||||
|
||||
import type { ImportResult } from '../../data.types';
|
||||
import type { ImportOptions } from '../../data.import';
|
||||
import { importVendorData } from '../../data.import';
|
||||
import { parseTypingMindFile } from './typingmind.parser';
|
||||
import { transformTypingMindToConversations } from './typingmind.transformer';
|
||||
|
||||
|
||||
/**
|
||||
* Import TypingMind data from a file
|
||||
*/
|
||||
export async function importTypingMindData(
|
||||
file: File,
|
||||
options: ImportOptions = {},
|
||||
): Promise<ImportResult> {
|
||||
return importVendorData(
|
||||
file,
|
||||
parseTypingMindFile,
|
||||
transformTypingMindToConversations,
|
||||
'typingmind',
|
||||
options,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
/**
|
||||
* TypingMind importer - main vendor implementation
|
||||
*/
|
||||
|
||||
import type { IDataVendor, ParseResult, TransformResult, ValidationResult } from '../vendor.types';
|
||||
import type { TypingMindExport } from './typingmind.schema';
|
||||
|
||||
import { parseTypingMindFile, validateTypingMindFile } from './typingmind.parser';
|
||||
import { transformTypingMindToConversations } from './typingmind.transformer';
|
||||
import { validateTypingMindSource } from './typingmind.validator';
|
||||
|
||||
|
||||
/**
|
||||
* TypingMind data import vendor
|
||||
*/
|
||||
export const TypingMindImporter: IDataVendor = {
|
||||
id: 'typingmind',
|
||||
label: 'TypingMind',
|
||||
description: 'Import conversations from TypingMind export files',
|
||||
|
||||
capabilities: {
|
||||
supportsFiles: true,
|
||||
supportsUrl: false,
|
||||
supportsText: false,
|
||||
},
|
||||
|
||||
async validateFile(file: File): Promise<boolean> {
|
||||
return validateTypingMindFile(file);
|
||||
},
|
||||
|
||||
async parseFile(file: File): Promise<ParseResult> {
|
||||
return parseTypingMindFile(file);
|
||||
},
|
||||
|
||||
async transformToConversations(parsedData: TypingMindExport): Promise<TransformResult> {
|
||||
return transformTypingMindToConversations(parsedData);
|
||||
},
|
||||
|
||||
validateSource(data: TypingMindExport): ValidationResult {
|
||||
return validateTypingMindSource(data);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,119 @@
|
||||
/**
|
||||
* Parser for TypingMind export files
|
||||
*/
|
||||
|
||||
import type { ParseResult } from '../vendor.types';
|
||||
import type { ImportWarning } from '../../data.types';
|
||||
import { typingMindExportSchema, type TypingMindExport } from './typingmind.schema';
|
||||
|
||||
|
||||
/**
|
||||
* Parse a TypingMind export file
|
||||
*/
|
||||
export async function parseTypingMindFile(file: File): Promise<ParseResult> {
|
||||
const warnings: ImportWarning[] = [];
|
||||
|
||||
try {
|
||||
// Read file content
|
||||
const fileContent = await file.text();
|
||||
|
||||
// Parse JSON
|
||||
let jsonData: any;
|
||||
try {
|
||||
jsonData = JSON.parse(fileContent);
|
||||
} catch (parseError) {
|
||||
return {
|
||||
success: false,
|
||||
data: null,
|
||||
warnings: [],
|
||||
error: `Invalid JSON: ${parseError instanceof Error ? parseError.message : 'Unknown error'}`,
|
||||
};
|
||||
}
|
||||
|
||||
// Validate against schema
|
||||
const parseResult = typingMindExportSchema.safeParse(jsonData);
|
||||
|
||||
if (!parseResult.success) {
|
||||
return {
|
||||
success: false,
|
||||
data: null,
|
||||
warnings: [],
|
||||
error: `Schema validation failed: ${parseResult.error.message}`,
|
||||
};
|
||||
}
|
||||
|
||||
const data = parseResult.data;
|
||||
|
||||
// Add warnings for optional features
|
||||
if (data.data.folders && data.data.folders.length > 0) {
|
||||
warnings.push({
|
||||
type: 'unsupported-feature',
|
||||
message: `Found ${data.data.folders.length} folders - folder structure will be preserved`,
|
||||
});
|
||||
}
|
||||
|
||||
if (data.data.userPrompts && data.data.userPrompts.length > 0) {
|
||||
warnings.push({
|
||||
type: 'unsupported-feature',
|
||||
message: `Found ${data.data.userPrompts.length} user prompts - these are not yet supported`,
|
||||
});
|
||||
}
|
||||
|
||||
if (data.data.userCharacters && data.data.userCharacters.length > 0) {
|
||||
warnings.push({
|
||||
type: 'unsupported-feature',
|
||||
message: `Found ${data.data.userCharacters.length} user characters - these are not yet supported`,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data,
|
||||
warnings,
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
data: null,
|
||||
warnings: [],
|
||||
error: `Failed to parse file: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Validate that a file appears to be a TypingMind export
|
||||
*/
|
||||
export async function validateTypingMindFile(file: File): Promise<boolean> {
|
||||
try {
|
||||
// Check file extension
|
||||
if (!file.name.endsWith('.json')) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check file size (must be reasonable)
|
||||
if (file.size === 0 || file.size > 100 * 1024 * 1024) { // Max 100MB
|
||||
return false;
|
||||
}
|
||||
|
||||
// Quick check: try to parse and look for TypingMind structure
|
||||
const content = await file.text();
|
||||
const json = JSON.parse(content);
|
||||
|
||||
// Look for TypingMind-specific structure
|
||||
return (
|
||||
json &&
|
||||
typeof json === 'object' &&
|
||||
'data' in json &&
|
||||
json.data &&
|
||||
typeof json.data === 'object' &&
|
||||
'chats' in json.data &&
|
||||
Array.isArray(json.data.chats)
|
||||
);
|
||||
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
/**
|
||||
* Zod schemas for TypingMind export format
|
||||
* Using relaxed parsing with .passthrough() to allow schema evolution
|
||||
*/
|
||||
|
||||
import * as z from 'zod/v4';
|
||||
|
||||
|
||||
/**
|
||||
* Message content part (text, image, etc.)
|
||||
*/
|
||||
const typingMindContentPartSchema = z.object({
|
||||
type: z.string(),
|
||||
text: z.string().optional(),
|
||||
image_url: z.any().optional(),
|
||||
}).passthrough();
|
||||
|
||||
|
||||
/**
|
||||
* Message schema
|
||||
* Content can be either a string or an array of content parts
|
||||
*/
|
||||
const typingMindMessageSchema = z.object({
|
||||
role: z.enum(['user', 'assistant', 'system']),
|
||||
content: z.union([
|
||||
z.string(),
|
||||
z.array(typingMindContentPartSchema),
|
||||
]),
|
||||
uuid: z.string().optional(),
|
||||
createdAt: z.string().optional(),
|
||||
updatedAt: z.string().optional(),
|
||||
}).passthrough();
|
||||
|
||||
|
||||
/**
|
||||
* Chat/Conversation schema
|
||||
*/
|
||||
const typingMindChatSchema = z.object({
|
||||
chatID: z.string(),
|
||||
chatTitle: z.string().optional(),
|
||||
messages: z.array(typingMindMessageSchema),
|
||||
createdAt: z.string().optional(),
|
||||
updatedAt: z.string().optional(),
|
||||
folderID: z.string().optional(),
|
||||
model: z.string().optional(),
|
||||
}).passthrough();
|
||||
|
||||
|
||||
/**
|
||||
* Folder schema
|
||||
*/
|
||||
const typingMindFolderSchema = z.object({
|
||||
id: z.string(),
|
||||
name: z.string(),
|
||||
color: z.string().optional(),
|
||||
createdAt: z.string().optional(),
|
||||
parentID: z.string().optional(),
|
||||
}).passthrough();
|
||||
|
||||
|
||||
/**
|
||||
* User prompt schema (custom instructions)
|
||||
*/
|
||||
const typingMindUserPromptSchema = z.object({
|
||||
id: z.string(),
|
||||
name: z.string(),
|
||||
prompt: z.string(),
|
||||
createdAt: z.string().optional(),
|
||||
}).passthrough();
|
||||
|
||||
|
||||
/**
|
||||
* User character schema (personas)
|
||||
*/
|
||||
const typingMindUserCharacterSchema = z.object({
|
||||
id: z.string(),
|
||||
name: z.string(),
|
||||
description: z.string().optional(),
|
||||
prompt: z.string().optional(),
|
||||
createdAt: z.string().optional(),
|
||||
}).passthrough();
|
||||
|
||||
|
||||
/**
|
||||
* Main TypingMind export schema
|
||||
*/
|
||||
export const typingMindExportSchema = z.object({
|
||||
data: z.object({
|
||||
chats: z.array(typingMindChatSchema),
|
||||
folders: z.array(typingMindFolderSchema).optional(),
|
||||
userPrompts: z.array(typingMindUserPromptSchema).optional(),
|
||||
userCharacters: z.array(typingMindUserCharacterSchema).optional(),
|
||||
}).passthrough(),
|
||||
}).passthrough();
|
||||
|
||||
|
||||
/**
|
||||
* TypeScript types inferred from schemas
|
||||
*/
|
||||
export type TypingMindExport = z.infer<typeof typingMindExportSchema>;
|
||||
export type TypingMindChat = z.infer<typeof typingMindChatSchema>;
|
||||
export type TypingMindMessage = z.infer<typeof typingMindMessageSchema>;
|
||||
export type TypingMindFolder = z.infer<typeof typingMindFolderSchema>;
|
||||
export type TypingMindUserPrompt = z.infer<typeof typingMindUserPromptSchema>;
|
||||
export type TypingMindUserCharacter = z.infer<typeof typingMindUserCharacterSchema>;
|
||||
@@ -0,0 +1,166 @@
|
||||
/**
|
||||
* Transform TypingMind data to Big-AGI format
|
||||
*/
|
||||
|
||||
import type { DConversation } from '~/common/stores/chat/chat.conversation';
|
||||
import type { DMessage, DMessageRole } from '~/common/stores/chat/chat.message';
|
||||
import type { TransformResult } from '../vendor.types';
|
||||
import type { ImportWarning } from '../../data.types';
|
||||
import type { TypingMindExport, TypingMindChat, TypingMindMessage } from './typingmind.schema';
|
||||
|
||||
import { createImportedConversation, createImportedMessage, normalizeMessageContent } from '../../pipeline/import.transformer';
|
||||
import { deduplicateMessages, sortMessagesByTimestamp } from '../../pipeline/import.transformer';
|
||||
|
||||
|
||||
/**
|
||||
* Transform TypingMind export to Big-AGI conversations
|
||||
*/
|
||||
export async function transformTypingMindToConversations(
|
||||
data: TypingMindExport,
|
||||
): Promise<TransformResult> {
|
||||
const warnings: ImportWarning[] = [];
|
||||
const unsupportedFeatures: string[] = [];
|
||||
const conversations: DConversation[] = [];
|
||||
|
||||
// Track unsupported features
|
||||
if (data.data.userPrompts?.length) {
|
||||
unsupportedFeatures.push('Custom user prompts');
|
||||
}
|
||||
if (data.data.userCharacters?.length) {
|
||||
unsupportedFeatures.push('User characters/personas');
|
||||
}
|
||||
|
||||
// Transform chats
|
||||
for (const chat of data.data.chats) {
|
||||
try {
|
||||
const conversation = transformTypingMindChat(chat, warnings);
|
||||
if (conversation && conversation.messages.length > 0) {
|
||||
conversations.push(conversation);
|
||||
} else {
|
||||
warnings.push({
|
||||
type: 'data-loss',
|
||||
message: `Skipped empty chat: ${chat.chatTitle || chat.chatID}`,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
warnings.push({
|
||||
type: 'data-loss',
|
||||
message: `Failed to transform chat ${chat.chatID}: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
conversations,
|
||||
warnings,
|
||||
unsupportedFeatures,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Transform a single TypingMind chat to a DConversation
|
||||
*/
|
||||
function transformTypingMindChat(
|
||||
chat: TypingMindChat,
|
||||
warnings: ImportWarning[],
|
||||
): DConversation {
|
||||
// Create conversation
|
||||
const conversation = createImportedConversation(
|
||||
chat.chatTitle || 'Untitled Chat',
|
||||
chat.chatID,
|
||||
chat.createdAt,
|
||||
chat.updatedAt,
|
||||
);
|
||||
|
||||
// Transform messages
|
||||
const messages: DMessage[] = [];
|
||||
for (const tmMessage of chat.messages) {
|
||||
try {
|
||||
const message = transformTypingMindMessage(tmMessage);
|
||||
if (message) {
|
||||
messages.push(message);
|
||||
}
|
||||
} catch (error) {
|
||||
warnings.push({
|
||||
type: 'data-loss',
|
||||
message: `Failed to transform message in chat ${chat.chatID}`,
|
||||
affectedConversationIds: [chat.chatID],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Deduplicate and sort messages
|
||||
conversation.messages = sortMessagesByTimestamp(deduplicateMessages(messages));
|
||||
|
||||
// Store folder reference in metadata if present
|
||||
if (chat.folderID) {
|
||||
if (!(conversation as any).metadata) {
|
||||
(conversation as any).metadata = {};
|
||||
}
|
||||
(conversation as any).metadata.typingmind = {
|
||||
folderId: chat.folderID,
|
||||
model: chat.model,
|
||||
};
|
||||
}
|
||||
|
||||
return conversation;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Transform a single TypingMind message to a DMessage
|
||||
*/
|
||||
function transformTypingMindMessage(tmMessage: TypingMindMessage): DMessage | null {
|
||||
// Normalize content
|
||||
const content = normalizeMessageContent(tmMessage.content);
|
||||
|
||||
if (!content || content.trim().length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Map role
|
||||
const role: DMessageRole = tmMessage.role === 'system'
|
||||
? 'system'
|
||||
: tmMessage.role === 'assistant'
|
||||
? 'assistant'
|
||||
: 'user';
|
||||
|
||||
// Create message
|
||||
const message = createImportedMessage(
|
||||
role,
|
||||
content,
|
||||
tmMessage.uuid,
|
||||
tmMessage.createdAt,
|
||||
);
|
||||
|
||||
return message;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Get statistics about the transform
|
||||
*/
|
||||
export function getTransformStats(result: TransformResult) {
|
||||
const totalMessages = result.conversations.reduce(
|
||||
(sum, conv) => sum + conv.messages.length,
|
||||
0,
|
||||
);
|
||||
|
||||
return {
|
||||
conversationsImported: result.conversations.length,
|
||||
messagesImported: totalMessages,
|
||||
foldersImported: 0, // Not yet supported
|
||||
charactersImported: result.conversations.reduce(
|
||||
(sum, conv) => sum + conv.messages.reduce(
|
||||
(msgSum, msg) => msgSum + msg.fragments.reduce(
|
||||
(fragSum, frag) => fragSum + ((frag as any).part?.text?.length || 0),
|
||||
0,
|
||||
),
|
||||
0,
|
||||
),
|
||||
0,
|
||||
),
|
||||
unsupportedItemsSkipped: result.unsupportedFeatures.length,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
# TypingMind Import - Unsupported Features
|
||||
|
||||
This document lists features from TypingMind exports that are not currently supported in Big-AGI import.
|
||||
|
||||
## Currently Unsupported
|
||||
|
||||
### 1. User Prompts / Custom Instructions
|
||||
- **TypingMind Feature**: Custom user prompts and instructions
|
||||
- **Status**: Not imported
|
||||
- **Workaround**: Manually recreate in Big-AGI settings or personas
|
||||
- **Future**: May be mapped to Big-AGI personas system
|
||||
|
||||
### 2. User Characters / Personas
|
||||
- **TypingMind Feature**: User-created characters with custom personalities
|
||||
- **Status**: Not imported
|
||||
- **Workaround**: Manually recreate using Big-AGI personas feature
|
||||
- **Future**: Should map to Big-AGI personas system
|
||||
|
||||
### 3. Folder Hierarchy
|
||||
- **TypingMind Feature**: Nested folder organization
|
||||
- **Status**: Folder IDs are preserved in metadata but not displayed
|
||||
- **Workaround**: Folder references are stored and can be used for future organization
|
||||
- **Future**: Will be mapped to Big-AGI folder system
|
||||
|
||||
### 4. Model Preferences
|
||||
- **TypingMind Feature**: Per-chat model selection
|
||||
- **Status**: Model information is preserved in metadata but not applied
|
||||
- **Workaround**: Manually select models in Big-AGI
|
||||
- **Future**: May restore model preferences
|
||||
|
||||
### 5. Attachments / Images
|
||||
- **TypingMind Feature**: Image and file attachments in messages
|
||||
- **Status**: Image URLs are lost in content normalization
|
||||
- **Workaround**: Images must be re-uploaded
|
||||
- **Future**: Should support image reference preservation
|
||||
|
||||
### 6. Message Metadata
|
||||
- **TypingMind Feature**: Token counts, costs, timing data
|
||||
- **Status**: Not preserved
|
||||
- **Workaround**: Big-AGI will recalculate token counts
|
||||
- **Future**: May preserve historical metadata
|
||||
|
||||
## Partially Supported
|
||||
|
||||
### 1. Message Content
|
||||
- **Text content**: Fully supported
|
||||
- **Multi-part content**: Text parts are concatenated
|
||||
- **Code blocks**: Preserved as-is
|
||||
- **Formatting**: Markdown formatting preserved
|
||||
|
||||
### 2. Timestamps
|
||||
- **Created timestamps**: Fully supported
|
||||
- **Updated timestamps**: Converted to conversation update time
|
||||
- **Timezone handling**: Preserved via ISO 8601 format
|
||||
|
||||
### 3. Message Roles
|
||||
- **User messages**: Fully supported
|
||||
- **Assistant messages**: Fully supported
|
||||
- **System messages**: Fully supported
|
||||
- **Function/tool messages**: Not present in TypingMind exports
|
||||
|
||||
## Fully Supported
|
||||
|
||||
### 1. Conversations
|
||||
- Chat ID preservation
|
||||
- Chat titles
|
||||
- Creation and update timestamps
|
||||
- Message history
|
||||
|
||||
### 2. Messages
|
||||
- Role (user/assistant/system)
|
||||
- Text content
|
||||
- Message order
|
||||
- UUIDs
|
||||
|
||||
### 3. Data Lineage
|
||||
- Original file hash tracking
|
||||
- Re-import detection
|
||||
- Original ID preservation
|
||||
|
||||
## Migration Notes
|
||||
|
||||
When migrating from TypingMind to Big-AGI:
|
||||
|
||||
1. **Export your data** from TypingMind settings
|
||||
2. **Review custom prompts** and recreate as Big-AGI personas
|
||||
3. **Note folder structure** - consider recreating in Big-AGI
|
||||
4. **Import the JSON file** - conversations will be preserved
|
||||
5. **Review imported chats** - verify content and organization
|
||||
|
||||
## Reporting Issues
|
||||
|
||||
If you encounter issues with TypingMind imports:
|
||||
|
||||
1. Check that your export file is valid JSON
|
||||
2. Ensure you're using the latest TypingMind export format
|
||||
3. Review the import warnings and errors
|
||||
4. Report issues with sample data (remove sensitive content)
|
||||
@@ -0,0 +1,68 @@
|
||||
/**
|
||||
* Validator for TypingMind source data
|
||||
*/
|
||||
|
||||
import type { ValidationResult } from '../vendor.types';
|
||||
import type { TypingMindExport } from './typingmind.schema';
|
||||
|
||||
|
||||
/**
|
||||
* Validate TypingMind source data before transformation
|
||||
*/
|
||||
export function validateTypingMindSource(data: TypingMindExport): ValidationResult {
|
||||
const errors: string[] = [];
|
||||
const warnings: string[] = [];
|
||||
|
||||
// Check required structure
|
||||
if (!data.data) {
|
||||
errors.push('Missing data object');
|
||||
return { valid: false, errors, warnings };
|
||||
}
|
||||
|
||||
if (!data.data.chats || !Array.isArray(data.data.chats)) {
|
||||
errors.push('Missing or invalid chats array');
|
||||
return { valid: false, errors, warnings };
|
||||
}
|
||||
|
||||
// Check if empty
|
||||
if (data.data.chats.length === 0) {
|
||||
warnings.push('No chats found in export');
|
||||
}
|
||||
|
||||
// Validate individual chats
|
||||
for (let i = 0; i < data.data.chats.length; i++) {
|
||||
const chat = data.data.chats[i];
|
||||
|
||||
if (!chat.chatID) {
|
||||
errors.push(`Chat at index ${i} is missing chatID`);
|
||||
}
|
||||
|
||||
if (!chat.messages || !Array.isArray(chat.messages)) {
|
||||
errors.push(`Chat ${chat.chatID || i} has invalid messages array`);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (chat.messages.length === 0) {
|
||||
warnings.push(`Chat ${chat.chatTitle || chat.chatID} is empty`);
|
||||
}
|
||||
|
||||
// Validate messages
|
||||
for (let j = 0; j < chat.messages.length; j++) {
|
||||
const message = chat.messages[j];
|
||||
|
||||
if (!message.role) {
|
||||
errors.push(`Message ${j} in chat ${chat.chatID} is missing role`);
|
||||
}
|
||||
|
||||
if (!message.content) {
|
||||
warnings.push(`Message ${j} in chat ${chat.chatID} has empty content`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
valid: errors.length === 0,
|
||||
errors,
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
/**
|
||||
* Registry for data import vendors
|
||||
*/
|
||||
|
||||
import type { IDataVendor, VendorId } from './vendor.types';
|
||||
|
||||
|
||||
/**
|
||||
* Global registry of data import vendors
|
||||
*/
|
||||
const _vendorRegistry = new Map<VendorId, IDataVendor>();
|
||||
|
||||
|
||||
/**
|
||||
* Register a data import vendor
|
||||
*/
|
||||
export function registerDataVendor(vendor: IDataVendor): void {
|
||||
if (_vendorRegistry.has(vendor.id)) {
|
||||
console.warn(`[Data Import] Vendor ${vendor.id} is already registered`);
|
||||
return;
|
||||
}
|
||||
_vendorRegistry.set(vendor.id, vendor);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Get a vendor by ID
|
||||
*/
|
||||
export function getDataVendor(vendorId: VendorId): IDataVendor | null {
|
||||
return _vendorRegistry.get(vendorId) || null;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Get all registered vendors
|
||||
*/
|
||||
export function getAllDataVendors(): IDataVendor[] {
|
||||
return Array.from(_vendorRegistry.values());
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Check if a vendor is registered
|
||||
*/
|
||||
export function hasDataVendor(vendorId: VendorId): boolean {
|
||||
return _vendorRegistry.has(vendorId);
|
||||
}
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
/**
|
||||
* Base types for data import vendors
|
||||
*/
|
||||
|
||||
import type { DConversation } from '~/common/stores/chat/chat.conversation';
|
||||
import type { ImportResult, ImportWarning } from '../data.types';
|
||||
|
||||
|
||||
/**
|
||||
* Vendor ID - unique identifier for each import source
|
||||
*/
|
||||
export type VendorId = 'typingmind' | 'chatgpt' | 'bigagi';
|
||||
|
||||
|
||||
/**
|
||||
* Base interface for all data import vendors
|
||||
*/
|
||||
export interface IDataVendor {
|
||||
// Vendor metadata
|
||||
readonly id: VendorId;
|
||||
readonly label: string;
|
||||
readonly description: string;
|
||||
|
||||
// Capabilities
|
||||
readonly capabilities: {
|
||||
supportsFiles: boolean;
|
||||
supportsUrl: boolean;
|
||||
supportsText: boolean;
|
||||
};
|
||||
|
||||
// File validation
|
||||
validateFile?(file: File): Promise<boolean>;
|
||||
|
||||
// Parse and transform
|
||||
parseFile(file: File): Promise<ParseResult>;
|
||||
transformToConversations(parsedData: any): Promise<TransformResult>;
|
||||
|
||||
// Optional: validate source data before transformation
|
||||
validateSource?(data: any): ValidationResult;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Result of parsing a file
|
||||
*/
|
||||
export interface ParseResult {
|
||||
success: boolean;
|
||||
data: any;
|
||||
warnings: ImportWarning[];
|
||||
error?: string;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Result of transforming parsed data to conversations
|
||||
*/
|
||||
export interface TransformResult {
|
||||
conversations: DConversation[];
|
||||
warnings: ImportWarning[];
|
||||
unsupportedFeatures: string[];
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Result of validating source data
|
||||
*/
|
||||
export interface ValidationResult {
|
||||
valid: boolean;
|
||||
errors: string[];
|
||||
warnings: string[];
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import * as React from 'react';
|
||||
|
||||
import { Box, Button, FormControl, Input, Sheet, Textarea, Typography } from '@mui/joy';
|
||||
import FileUploadIcon from '@mui/icons-material/FileUpload';
|
||||
import CloudUploadIcon from '@mui/icons-material/CloudUpload';
|
||||
|
||||
import { Brand } from '~/common/app.config';
|
||||
import { FormRadioOption } from '~/common/components/forms/FormRadioControl';
|
||||
@@ -20,6 +21,7 @@ import { importConversationsFromFilesAtRest, openConversationsAtRestPicker } fro
|
||||
|
||||
import { FlashRestore } from './BackupRestore';
|
||||
import { ImportedOutcome, ImportOutcomeModal } from './ImportOutcomeModal';
|
||||
import { DataImportModal } from '~/modules/data/ui/DataImportModal';
|
||||
|
||||
|
||||
export type ImportConfig = { dir: 'import' };
|
||||
@@ -43,6 +45,7 @@ export function ImportChats(props: { onConversationActivate: (conversationId: DC
|
||||
const [chatGptSource, setChatGptSource] = React.useState('');
|
||||
const [importJson, setImportJson] = React.useState<string | null>(null);
|
||||
const [importOutcome, setImportOutcome] = React.useState<ImportedOutcome | null>(null);
|
||||
const [typingMindModalOpen, setTypingMindModalOpen] = React.useState(false);
|
||||
|
||||
// derived state
|
||||
const isUrl = importMedia === 'link';
|
||||
@@ -150,6 +153,13 @@ export function ImportChats(props: { onConversationActivate: (conversationId: DC
|
||||
</Button>
|
||||
)}
|
||||
|
||||
<Button
|
||||
variant='soft' endDecorator={<CloudUploadIcon />} sx={{ minWidth: 240, justifyContent: 'space-between' }}
|
||||
onClick={() => setTypingMindModalOpen(true)}
|
||||
>
|
||||
TypingMind · Export
|
||||
</Button>
|
||||
|
||||
{/* Insert to Restore a Flash */}
|
||||
<FlashRestore unlockRestore={true} />
|
||||
|
||||
@@ -193,5 +203,14 @@ export function ImportChats(props: { onConversationActivate: (conversationId: DC
|
||||
{/* import outcome */}
|
||||
{!!importOutcome && <ImportOutcomeModal outcome={importOutcome} rawJson={importJson} onClose={handleImportOutcomeClosed} />}
|
||||
|
||||
{/* TypingMind import modal */}
|
||||
<DataImportModal
|
||||
vendorId='typingmind'
|
||||
vendorLabel='TypingMind'
|
||||
open={typingMindModalOpen}
|
||||
onConversationActivate={props.onConversationActivate}
|
||||
onClose={() => setTypingMindModalOpen(false)}
|
||||
/>
|
||||
|
||||
</>;
|
||||
}
|
||||
Reference in New Issue
Block a user