iVProg/js/assessment/output_matching/output_matching.js

import { Decimal } from 'decimal.js';
import { InputAssessment } from "../../util/input_assessment";
import { OutputTest } from "../../util/outputTest";
import { Config } from "../../util/config";
import { levenshteinDistance } from "../../util/utils";
import { OutputAssessmentResult } from './assessment_result';
import * as TypeParser from "./../../typeSystem/parsers";
import * as LocalizedStringsService from "../../services/localizedStringsService";
import * as OutputResult from "./output_result";

const LocalizedStrings = LocalizedStringsService.getInstance();

export class OutputMatching {

  static get NUM_REGEX () {
    return /^[+-]?([0-9]+([.][0-9]*)?(e[+-]?[0-9]+)?)$/;
  } 

  static get NUM_IN_STRING_REGEX () {
    return /[+-]?([0-9]+([.][0-9]*)?(e[+-]?[0-9]+)?)/g;
  }

  static get BOOLEAN_REGEX () {
    const str = `^(${LocalizedStrings.getUI("logic_value_true")}|${LocalizedStrings.getUI("logic_value_false")})$`;
    return new RegExp(str);
  }

  static get BOOLEAN_IN_STRING_REGEX () {
    const str = `(${LocalizedStrings.getUI("logic_value_true")}|${LocalizedStrings.getUI("logic_value_false")})`;
    return new RegExp(str, 'g');
  }
  
  constructor (program, input_list, expected_output, test_name) {
    this.program = program;
    this.name = test_name;
    this.input_list = input_list;
    this.expected_output = expected_output;
  }

  eval () {
    const refThis = this;
    const input = new InputAssessment(this.input_list);
    const gen_output = new OutputTest();
    this.program.registerInput(input);
    this.program.registerOutput(gen_output);
    const start_time = Date.now();
    return this.program.interpretAST().then( sto => {
      const final_time = Date.now() - start_time;
      if(input.isInputAvailable()) {
        return new OutputAssessmentResult(this.name, 1, input.input_list,
          null, sto, final_time, refThis.getErrorMessage('test_case_few_reads', this.name+1))
      }
      const result = gen_output.list.map((g_out, i) => {
        if(i >= this.expected_output.length) {
          return new OutputResult.OutputMatchResult(null, g_out, 0, this.getPotentialOutputType(g_out));
        }
        return this.outputMatch(g_out, this.expected_output[i]);
      }, this);
      if(this.expected_output.length > gen_output.list.length) {
        for(let i = gen_output.list.length; i < this.expected_output.length; ++i) {
          const e_out = this.expected_output[i];
          result.push(new OutputResult.OutputMatchResult(e_out, null, 0, this.getPotentialOutputType(e_out)));
        }
      }
      return new OutputAssessmentResult(this.name, 0,  input.input_list, result, sto, final_time);
    }).catch(error => {
      return new OutputAssessmentResult(this.name, 1,  input.input_list, null, null,
        null, refThis.getErrorMessage('test_case_exception', this.name + 1, error.message))
    });
  }

  getPotentialOutputType (output) {
    if(OutputMatching.NUM_REGEX.test(output)) {
      return "number";
    } else if (OutputMatching.BOOLEAN_REGEX.test(output)) {
      return "bool";
    } else {
      return "string";
    }
  }

  outputMatch (g_output, e_output) {
    if(OutputMatching.NUM_REGEX.test(e_output)) {
      if(!OutputMatching.NUM_REGEX.test(g_output)) {
        return this.checkStrings(g_output, e_output);
      }
      const g_num = new Decimal(g_output);
      const e_num = new Decimal(e_output);
      return this.checkNumbers(g_num, e_num);
    } else if (OutputMatching.BOOLEAN_REGEX.test(e_output)) {
      if (!OutputMatching.BOOLEAN_REGEX.test(g_output)) {
        return this.checkStrings(g_output, e_output)
      }
      const g_bool = TypeParser.toBool(g_output);
      const e_bool = TypeParser.toBool(e_output);
      return this.checkBoolean(g_bool, e_bool);
    } else {
      return this.checkStrings(g_output, e_output);
    }
  }

  checkNumbers (g_num, e_num) {
    const decimalPlaces = Math.min(e_num.dp(), Config.decimalPlaces);
    g_num = new Decimal(g_num.toFixed(decimalPlaces, Decimal.ROUND_FLOOR));
    e_num = new Decimal(e_num.toFixed(decimalPlaces, Decimal.ROUND_FLOOR));
    const result = g_num.eq(e_num);
    const grade = result ? 1 : 0;
    return OutputResult.createNumberResult(e_num.toNumber(), g_num.toNumber(), grade);
  }

  checkBoolean (g_bool, e_bool) {
    const grade = g_bool == e_bool ? 1 : 0;
    const g_bool_text = TypeParser.convertBoolToString(g_bool);
    const e_bool_text = TypeParser.convertBoolToString(e_bool);
    return OutputResult.createBoolResult(e_bool_text, g_bool_text, grade);
  }

  checkStrings (g_output, e_ouput) {
    const assessmentList = []
    let e_output_clean = e_ouput;
    let g_output_clean = g_output;
    if (OutputMatching.NUM_IN_STRING_REGEX.test(e_ouput)) {
      const expected_numbers = e_ouput.match(OutputMatching.NUM_IN_STRING_REGEX);
      const generated_numbers = g_output.match(OutputMatching.NUM_IN_STRING_REGEX) || [];
      const result = generated_numbers.map((val, i) => {
        if(i >= expected_numbers.length) {
          return OutputResult.createNumberResult(null, val, 0);
        }
        const g_val = new Decimal(val)
        const e_val = new Decimal(expected_numbers[i]);
        return this.checkNumbers(g_val, e_val);
      }, this);
      if(expected_numbers.length > generated_numbers.length) {
        for(let i = generated_numbers.length; i < expected_numbers.length; ++i) {
          result.push(OutputResult.createNumberResult(expected_numbers[i], null, 0));
        }
      }
      e_output_clean = e_output_clean.replace(OutputMatching.NUM_IN_STRING_REGEX, '').trim();
      g_output_clean = g_output_clean.replace(OutputMatching.NUM_IN_STRING_REGEX, '').trim();
      const numberGrade = result.reduce((prev, r) => prev + r.grade, 0) / result.length;
      assessmentList.push(numberGrade);
    } 
    if(OutputMatching.BOOLEAN_IN_STRING_REGEX.test(e_ouput)) {
      const expected_bools = e_ouput.match(OutputMatching.BOOLEAN_IN_STRING_REGEX);
      const generated_bools = g_output.match(OutputMatching.BOOLEAN_IN_STRING_REGEX) || [];
      const result = generated_bools.map((val, i) => {
        if(i >= expected_bools.length) {
          return OutputResult.createBoolResult(null, val, 0);
        }
        const g_bool = TypeParser.toBool(val);
        const e_bool = TypeParser.toBool(expected_bools[i]);
        return this.checkBoolean(g_bool, e_bool );
      }, this);
      if(expected_bools.length > generated_bools.length) {
        for(let i = generated_bools.length; i < expected_bools.length; ++i) {
          result.push(OutputResult.createBoolResult(expected_bools[i], null, 0));
        }
      }
      e_output_clean = e_output_clean.replace(OutputMatching.BOOLEAN_IN_STRING_REGEX, '').trim();
      g_output_clean = g_output_clean.replace(OutputMatching.BOOLEAN_IN_STRING_REGEX, '').trim();
      const boolGrade = result.reduce((prev, r) => prev + r.grade, 0) / result.length;
      assessmentList.push(boolGrade);
    }
    const dist = levenshteinDistance(g_output_clean, e_output_clean);
    let gradeDiff = Math.max(0, e_output_clean.length - dist)/e_output_clean.length;
    gradeDiff = Number.isNaN(gradeDiff) ? 0 : gradeDiff;
    const assessment_size = assessmentList.length + 1;
    const gradeAcc = assessmentList.reduce((prev, val) => prev + val/assessment_size, 0);
    const finalGrade = 1 * (gradeDiff/assessment_size + gradeAcc);
    return OutputResult.createStringResult(e_ouput, g_output, finalGrade);
  }
  
  getErrorMessage (errorID, ...args) {
    return LocalizedStrings.getError(errorID, args);
  }
}