codegraph/__tests__/evaluation/fixtures/python-project/ground-truth.ts

/**
 * Ground truth definitions for the Python task management fixture
 */

import { FixtureGroundTruth } from '../../types';

export const pythonFixture: FixtureGroundTruth = {
  name: 'python-taskmanager',
  path: '__tests__/evaluation/fixtures/python-project',
  language: 'python',
  totalFiles: 5,
  approximateTokens: 1200, // Rough estimate

  testCases: [
    // =========================================================================
    // Search Tests
    // =========================================================================
    {
      id: 'py-search-auth',
      description: 'Search for authentication functionality',
      query: 'authentication login',
      type: 'search',
      expectedSymbols: ['AuthService', 'AuthService.login', 'AuthService.register', 'verify_password'],
      irrelevantSymbols: ['TaskService', 'validate_task_title', 'Project'],
      minRecall: 0.7,
      minPrecision: 0.5,
    },
    {
      id: 'py-search-task',
      description: 'Search for task management',
      query: 'task create complete',
      type: 'search',
      expectedSymbols: ['TaskService', 'TaskService.create_task', 'TaskService.complete_task', 'Task'],
      irrelevantSymbols: ['AuthService', 'validate_email', 'hash_password'],
      minRecall: 0.7,
      minPrecision: 0.5,
    },
    {
      id: 'py-search-validation',
      description: 'Search for validation',
      query: 'validate',
      type: 'search',
      expectedSymbols: ['validate_email', 'validate_password', 'validate_task_title'],
      irrelevantSymbols: ['hash_password', 'generate_token', 'TaskService'],
      minRecall: 0.8,
      minPrecision: 0.6,
    },

    // =========================================================================
    // Context Tests
    // =========================================================================
    {
      id: 'py-context-login-bug',
      description: 'Build context for fixing login issues',
      query: 'debug why users cannot log in',
      type: 'context',
      expectedSymbols: [
        'AuthService.login',
        'verify_password',
        'db.get_user_by_email',
        'User',
        'hash_password',
      ],
      irrelevantSymbols: [
        'TaskService',
        'validate_task_title',
        'Project',
        'Task',
      ],
      minRecall: 0.8,
      minPrecision: 0.6,
    },
    {
      id: 'py-context-task-creation',
      description: 'Build context for task creation flow',
      query: 'understand how tasks are created',
      type: 'context',
      expectedSymbols: [
        'TaskService.create_task',
        'validate_task_title',
        'auth_service.get_user_id',
        'db.create_task',
        'Task',
        'generate_token',
      ],
      irrelevantSymbols: [
        'validate_email',
        'hash_password',
        'AuthService.register',
        'Project',
      ],
      minRecall: 0.7,
      minPrecision: 0.5,
    },
    {
      id: 'py-context-user-registration',
      description: 'Build context for user registration',
      query: 'add email confirmation to registration',
      type: 'context',
      expectedSymbols: [
        'AuthService.register',
        'validate_email',
        'validate_password',
        'hash_password',
        'db.create_user',
        'User',
      ],
      irrelevantSymbols: [
        'TaskService',
        'validate_task_title',
        'Task',
        'Project',
      ],
      minRecall: 0.7,
      minPrecision: 0.6,
    },

    // =========================================================================
    // Callers Tests
    // =========================================================================
    {
      id: 'py-callers-get_user_id',
      description: 'Find all callers of auth_service.get_user_id',
      query: 'get_user_id',
      type: 'callers',
      targetSymbol: 'get_user_id',
      expectedSymbols: [
        'TaskService.create_task',
        'TaskService.get_task',
        'TaskService.get_user_tasks',
      ],
      irrelevantSymbols: [
        'AuthService.login',
        'validate_email',
        'hash_password',
      ],
      minRecall: 1.0,
      minPrecision: 1.0,
    },
    {
      id: 'py-callers-validate_email',
      description: 'Find all callers of validate_email',
      query: 'validate_email',
      type: 'callers',
      targetSymbol: 'validate_email',
      expectedSymbols: [
        'AuthService.register',
      ],
      irrelevantSymbols: [
        'TaskService',
        'validate_password',
        'hash_password',
      ],
      minRecall: 1.0,
      minPrecision: 1.0,
    },
    {
      id: 'py-callers-generate_token',
      description: 'Find all callers of generate_token',
      query: 'generate_token',
      type: 'callers',
      targetSymbol: 'generate_token',
      expectedSymbols: [
        'AuthService.register',
        'AuthService.login',
        'TaskService.create_task',
      ],
      irrelevantSymbols: [
        'validate_email',
        'validate_password',
        'db.get_user',
      ],
      minRecall: 1.0,
      minPrecision: 1.0,
    },

    // =========================================================================
    // Callees Tests
    // =========================================================================
    {
      id: 'py-callees-login',
      description: 'Find what AuthService.login calls',
      query: 'login',
      type: 'callees',
      targetSymbol: 'login',
      expectedSymbols: [
        'db.get_user_by_email',
        'verify_password',
        'generate_token',
      ],
      irrelevantSymbols: [
        'validate_email',
        'hash_password',
        'validate_task_title',
      ],
      minRecall: 1.0,
      minPrecision: 1.0,
    },
    {
      id: 'py-callees-create_task',
      description: 'Find what TaskService.create_task calls',
      query: 'create_task',
      type: 'callees',
      targetSymbol: 'TaskService.create_task',
      expectedSymbols: [
        'auth_service.get_user_id',
        'validate_task_title',
        'generate_token',
        'db.create_task',
      ],
      irrelevantSymbols: [
        'validate_email',
        'hash_password',
        'db.get_user',
      ],
      minRecall: 0.8,
      minPrecision: 0.8,
    },

    // =========================================================================
    // Impact Tests
    // =========================================================================
    {
      id: 'py-impact-generate_token',
      description: 'Impact of changing generate_token',
      query: 'generate_token',
      type: 'impact',
      targetSymbol: 'generate_token',
      expectedSymbols: [
        // Direct callers
        'AuthService.register',
        'AuthService.login',
        'TaskService.create_task',
      ],
      irrelevantSymbols: [
        'validate_email',
        'validate_task_title',
        'db.get_project',
      ],
      minRecall: 0.8,
      minPrecision: 0.7,
    },
    {
      id: 'py-impact-get_user_id',
      description: 'Impact of changing get_user_id',
      query: 'get_user_id',
      type: 'impact',
      targetSymbol: 'get_user_id',
      expectedSymbols: [
        'TaskService.create_task',
        'TaskService.get_task',
        'TaskService.get_user_tasks',
        'TaskService.complete_task',
        'TaskService.delete_task',
      ],
      irrelevantSymbols: [
        'AuthService.register',
        'validate_email',
        'hash_password',
      ],
      minRecall: 0.8,
      minPrecision: 0.7,
    },
  ],

  // Known call graph edges for validation
  callGraph: [
    // Auth -> Database
    { caller: 'AuthService.register', callee: 'db.get_user_by_email' },
    { caller: 'AuthService.register', callee: 'db.create_user' },
    { caller: 'AuthService.login', callee: 'db.get_user_by_email' },

    // Auth -> Crypto
    { caller: 'AuthService.register', callee: 'hash_password' },
    { caller: 'AuthService.register', callee: 'generate_token' },
    { caller: 'AuthService.login', callee: 'verify_password' },
    { caller: 'AuthService.login', callee: 'generate_token' },

    // Auth -> Validation
    { caller: 'AuthService.register', callee: 'validate_email' },
    { caller: 'AuthService.register', callee: 'validate_password' },

    // Task -> Auth
    { caller: 'TaskService.create_task', callee: 'auth_service.get_user_id' },
    { caller: 'TaskService.get_task', callee: 'auth_service.get_user_id' },
    { caller: 'TaskService.get_user_tasks', callee: 'auth_service.get_user_id' },

    // Task -> Database
    { caller: 'TaskService.create_task', callee: 'db.create_task' },
    { caller: 'TaskService.get_task', callee: 'db.get_task' },
    { caller: 'TaskService.get_user_tasks', callee: 'db.get_user_tasks' },
    { caller: 'TaskService.complete_task', callee: 'db.update_task' },
    { caller: 'TaskService.delete_task', callee: 'db.delete_task' },

    // Task -> Crypto
    { caller: 'TaskService.create_task', callee: 'generate_token' },

    // Task -> Validation
    { caller: 'TaskService.create_task', callee: 'validate_task_title' },

    // Task -> Task (internal)
    { caller: 'TaskService.complete_task', callee: 'TaskService.get_task' },
    { caller: 'TaskService.delete_task', callee: 'TaskService.get_task' },
  ],
};