iVProg/js/assessment/output_matching/output_matching.js

import { Decimal } from "decimal.js";
import { InputAssessment } from "../../util/input_assessment";
import { OutputTest } from "../../util/outputTest";
import { Config } from "../../util/config";
import { levenshteinDistance } from "../../util/utils";
import { OutputAssessmentResult } from "./assessment_result";
import * as TypeParser from "./../../typeSystem/parsers";
import * as LocalizedStringsService from "../../services/localizedStringsService";
import * as OutputResult from "./output_result";

const LocalizedStrings = LocalizedStringsService.getInstance();

export class OutputMatching {
  static get NUM_REGEX () {
    return /^[+-]?([0-9]+([.][0-9]*)?(e[+-]?[0-9]+)?)$/;
  }

  static get NUM_IN_STRING_REGEX () {
    return /[+-]?([0-9]+([.][0-9]*)?(e[+-]?[0-9]+)?)/g;
  }

  static get BOOLEAN_REGEX () {
    const str = `^(${LocalizedStrings.getUI(
      "logic_value_true"
    )}|${LocalizedStrings.getUI("logic_value_false")})$`;
    return new RegExp(str);
  }

  static get BOOLEAN_IN_STRING_REGEX () {
    const str = `(${LocalizedStrings.getUI(
      "logic_value_true"
    )}|${LocalizedStrings.getUI("logic_value_false")})`;
    return new RegExp(str, "g");
  }

  constructor (program, input_list, expected_output, test_name, tag = null) {
    this.program = program;
    this.name = test_name;
    this.input_list = input_list;
    this.expected_output = expected_output;
    this.tag = tag;
  }

  eval () {
    const input = new InputAssessment(this.input_list);
    const gen_output = new OutputTest();
    this.program.registerInput(input);
    this.program.registerOutput(gen_output);
    const start_time = Date.now();
    return this.program
      .interpretAST()
      .then((sto) => {
        const final_time = Date.now() - start_time;
        if (input.isInputAvailable()) {
          const error = this.getErrorMessage(
            "test_case_few_reads",
            this.name + 1
          );
          return new OutputAssessmentResult(
            this.name,
            1,
            input.input_list,
            null,
            sto,
            final_time,
            this.tag,
            error,
            error.id
          );
        }
        const result = gen_output.list.map((g_out, i) => {
          if (i >= this.expected_output.length) {
            return new OutputResult.OutputMatchResult(
              null,
              g_out,
              0,
              this.getPotentialOutputType(g_out)
            );
          }
          return this.outputMatch(g_out, this.expected_output[i]);
        }, this);
        if (this.expected_output.length > gen_output.list.length) {
          console.log(
            "Saída insuficientes!",
            this.expected_output.length,
            gen_output.list.length
          );
          for (
            let i = gen_output.list.length;
            i < this.expected_output.length;
            ++i
          ) {
            const e_out = this.expected_output[i];
            result.push(
              new OutputResult.OutputMatchResult(
                e_out,
                null,
                0,
                this.getPotentialOutputType(e_out)
              )
            );
          }
        } else if (
          this.expected_output.length == 0 &&
          this.expected_output.length == gen_output.list.length
        ) {
          // no output expected....
          result.push(new OutputResult.OutputMatchResult("", "", 1, "string"));
        }
        return new OutputAssessmentResult(
          this.name,
          0,
          input.input_list,
          result,
          sto,
          final_time,
          this.tag
        );
      })
      .catch((error) => {
        return new OutputAssessmentResult(
          this.name,
          1,
          input.input_list,
          null,
          null,
          null,
          this.tags,
          this.getErrorMessage(
            "test_case_exception",
            this.name + 1,
            error.message
          ),
          error.id
        );
      });
  }

  getPotentialOutputType (output) {
    if (OutputMatching.NUM_REGEX.test(output)) {
      return "number";
    } else if (OutputMatching.BOOLEAN_REGEX.test(output)) {
      return "bool";
    } else {
      return "string";
    }
  }

  outputMatch (g_output, e_output) {
    if (OutputMatching.NUM_REGEX.test(e_output)) {
      if (!OutputMatching.NUM_REGEX.test(g_output)) {
        return this.checkStrings(g_output, e_output);
      }
      const g_num = new Decimal(g_output);
      const e_num = new Decimal(e_output);
      return this.checkNumbers(g_num, e_num);
    } else if (OutputMatching.BOOLEAN_REGEX.test(e_output)) {
      if (!OutputMatching.BOOLEAN_REGEX.test(g_output)) {
        return this.checkStrings(g_output, e_output);
      }
      const g_bool = TypeParser.toBool(g_output);
      const e_bool = TypeParser.toBool(e_output);
      return this.checkBoolean(g_bool, e_bool);
    } else {
      return this.checkStrings(g_output, e_output);
    }
  }

  checkNumbers (g_num, e_num) {
    const decimalPlaces = Math.min(e_num.dp(), Config.decimalPlaces);
    g_num = new Decimal(g_num.toFixed(decimalPlaces, Decimal.ROUND_FLOOR));
    e_num = new Decimal(e_num.toFixed(decimalPlaces, Decimal.ROUND_FLOOR));
    const result = g_num.eq(e_num);
    const grade = result ? 1 : 0;
    return OutputResult.createNumberResult(
      e_num.toNumber(),
      g_num.toNumber(),
      grade
    );
  }

  checkBoolean (g_bool, e_bool) {
    const grade = g_bool == e_bool ? 1 : 0;
    const g_bool_text = TypeParser.convertBoolToString(g_bool);
    const e_bool_text = TypeParser.convertBoolToString(e_bool);
    return OutputResult.createBoolResult(e_bool_text, g_bool_text, grade);
  }

  checkStrings (g_output, e_ouput) {
    const assessmentList = [];
    let e_output_clean = e_ouput.trim();
    let g_output_clean = g_output.trim();
    if (OutputMatching.NUM_IN_STRING_REGEX.test(e_ouput)) {
      const expected_numbers = e_ouput.match(
        OutputMatching.NUM_IN_STRING_REGEX
      );
      const generated_numbers =
        g_output.match(OutputMatching.NUM_IN_STRING_REGEX) || [];
      const result = generated_numbers.map((val, i) => {
        if (i >= expected_numbers.length) {
          return OutputResult.createNumberResult(null, val, 0);
        }
        const g_val = new Decimal(val);
        const e_val = new Decimal(expected_numbers[i]);
        return this.checkNumbers(g_val, e_val);
      }, this);
      if (expected_numbers.length > generated_numbers.length) {
        for (
          let i = generated_numbers.length;
          i < expected_numbers.length;
          ++i
        ) {
          result.push(
            OutputResult.createNumberResult(expected_numbers[i], null, 0)
          );
        }
      }
      e_output_clean = e_output_clean.replace(
        OutputMatching.NUM_IN_STRING_REGEX,
        ""
      );
      g_output_clean = g_output_clean.replace(
        OutputMatching.NUM_IN_STRING_REGEX,
        ""
      );
      const numberGrade =
        result.reduce((prev, r) => prev + r.grade, 0) / result.length;
      assessmentList.push(numberGrade);
    }
    if (OutputMatching.BOOLEAN_IN_STRING_REGEX.test(e_ouput)) {
      const expected_bools = e_ouput.match(
        OutputMatching.BOOLEAN_IN_STRING_REGEX
      );
      const generated_bools =
        g_output.match(OutputMatching.BOOLEAN_IN_STRING_REGEX) || [];
      const result = generated_bools.map((val, i) => {
        if (i >= expected_bools.length) {
          return OutputResult.createBoolResult(null, val, 0);
        }
        const g_bool = TypeParser.toBool(val);
        const e_bool = TypeParser.toBool(expected_bools[i]);
        return this.checkBoolean(g_bool, e_bool);
      }, this);
      if (expected_bools.length > generated_bools.length) {
        for (let i = generated_bools.length; i < expected_bools.length; ++i) {
          result.push(
            OutputResult.createBoolResult(expected_bools[i], null, 0)
          );
        }
      }
      e_output_clean = e_output_clean.replace(
        OutputMatching.BOOLEAN_IN_STRING_REGEX,
        ""
      );
      g_output_clean = g_output_clean.replace(
        OutputMatching.BOOLEAN_IN_STRING_REGEX,
        ""
      );
      const boolGrade =
        result.reduce((prev, r) => prev + r.grade, 0) / result.length;
      assessmentList.push(boolGrade);
    }
    const dist = levenshteinDistance(g_output_clean, e_output_clean);
    let gradeDiff =
      Math.max(0, e_output_clean.length - dist) / e_output_clean.length;
    gradeDiff = Number.isNaN(gradeDiff) ? 0 : gradeDiff;
    const assessment_size = assessmentList.length + 1;
    const gradeAcc = assessmentList.reduce(
      (prev, val) => prev + val / assessment_size,
      0
    );
    const finalGrade = 1 * (gradeDiff / assessment_size + gradeAcc);
    return OutputResult.createStringResult(e_ouput, g_output, finalGrade);
  }

  getErrorMessage (errorID, ...args) {
    return LocalizedStrings.getError(errorID, args);
  }
}