Implement instruction count (counts commands and expressions) to limit program execution and delay some computations

Implement an assessment config property to limit program execution during automated assessment

i18n/error.csv

@@ -132,4 +132,5 @@ exceeded_recursive_calls,Erro na execução da linha $0: Número de chamadas rec
 invalid_for_variable,Erro na linha $0: A variavel $1 utilizada no comando repita_para deve ser do tipo inteiro.,Error at line $0: The variable $1 used in the repeat_for command must be of type int.,
 invalid_for_from,Erro na linha $0: O valor $1 passado para o parâmetro 'de' do comando repita_para deve ser do tipo inteiro.,Error at line $0: The value $1 passed to the parameter 'from' of the repeat_for command must be of type int.,
 invalid_for_to,Erro na linha $0: O valor $1 passado para o parâmetro 'para' do comando repita_para deve ser do tipo inteiro.,Error at line $0: The value $1 passed to the parameter 'to' of the repeat_for command must be of type int.,
-invalid_for_pass,Erro na linha $0: O valor $1 passado para o parâmetro 'passo' do comando repita_para deve ser do tipo inteiro.,Error at line $0: The value $1 passed to the parameter 'pass' of the repeat_for command must be of type int.,
+invalid_for_pass,Erro na linha $0: O valor $1 passado para o parâmetro 'passo' do comando repita_para deve ser do tipo inteiro.,Error at line $0: The value $1 passed to the parameter 'pass' of the repeat_for command must be of type int.,
+exceed_max_instructions,Número de instruções excedeu o limite definido. Certifique-se que seu código não possui laços infinitos ou muitas chamadas de funções recursivas.,The number of instructions executed by your program exceed the defined limit. Check your code for infinite loops or excessive recursive functions calls.,

i18n/message.csv

@@ -8,4 +8,5 @@ assessment-empty-generated-tooltip,O programa não gerou saídas suficientes,The
 testcase_autogen_unused_input,O caso de teste $0 possui mais entradas do que as leituras feitas no programa.,The test case $0 has more inputs than output than the number of reads present in the algorithm.," "
 testcase_autogen_empty,O caso de teste $0 não gerou qualquer saída.,The test case $0 did not generate any output.," "
 success_execution,Programa executado com sucesso!,Program executed successfully!,
-aborted_execution,A execução do programa foi interrompida!,Program execution was aborted!,
+aborted_execution,A execução do programa foi interrompida!,Program execution was aborted!,
+unexpected_execution_error,Erro inesperado durante a execução do programa.,Unexpected error during program execution.,

js/assessment/ivprogAssessment.js

@@ -3,63 +3,72 @@ import { IVProgProcessor } from "./../processor/ivprogProcessor";
 import { DOMConsole} from "./../io/domConsole";
 import * as LocalizedStringsService from "../services/localizedStringsService";
 import { OutputMatching } from './output_matching/output_matching';
+import { Config } from '../util/config';
 
 
 const LocalizedStrings = LocalizedStringsService.getInstance();
 
 const StringTypes = line_i18n.StringTypes;
 
+const AssessmentConfig = {
+  max_instruction_count: 350250,
+  suspend_threshold: 200
+}
+
 export class IVProgAssessment {
 
   constructor (ast_code, testCases, domConsole) {
     this.ast_code = ast_code;
     this.testCases = testCases;
     this.domConsole = domConsole;
+    this.old_config = JSON.parse(JSON.stringify(Config));
+    Config.setConfig(AssessmentConfig);
   }
 
   runTest () {
-    const outerRef = this;
     try {
       // loop test cases and show messages through domconsole
       const partialTests = this.testCases.map( (t, name) => {
-        return new OutputMatching(new IVProgProcessor(outerRef.ast_code), t.input, t.output, name);
+        return new OutputMatching(new IVProgProcessor(this.ast_code), t.input, t.output, name);
       });
       const testResult = partialTests.map(om => om.eval());
       return Promise.all(testResult).then(results => {
         let grade = 0;
-        for(let i = 0; i < results.length; ++i) {
+        for(let i = 0; i < results.length; i += 1) {
           const result = results[i];
           grade += result.grade;
           if(result.grade == 1) {
-            outerRef.writeToConsole(DOMConsole.INFO, StringTypes.MESSAGE,'test_case_success',
+            this.writeToConsole(DOMConsole.INFO, StringTypes.MESSAGE,'test_case_success',
               result.name + 1, result.generateOutput());
           } else if (result.status == 1) {
-            outerRef.writeToConsole(DOMConsole.ERR, StringTypes.ERROR,'test_case_failed_exception',
+            this.writeToConsole(DOMConsole.ERR, StringTypes.ERROR,'test_case_failed_exception',
               result.name + 1, result.error_msg, result.generateOutput());
           } else {
-            outerRef.writeToConsole(DOMConsole.ERR, StringTypes.ERROR,'test_case_failed',
+            this.writeToConsole(DOMConsole.ERR, StringTypes.ERROR,'test_case_failed',
               result.name + 1, result.generateOutput());
           }
         }
         grade /= results.length;
         const channel = grade == 1 ? DOMConsole.INFO : DOMConsole.ERR;
-        outerRef.writeToConsole(channel, StringTypes.MESSAGE, "test_suite_grade", (grade * 100).toFixed(2));
+        this.writeToConsole(channel, StringTypes.MESSAGE, "test_suite_grade", (grade * 100).toFixed(2));
         return grade;
       });
       // return testResult.then(function (total) {
-      //   const grade = total / outerRef.testCases.length;
+      //   const grade = total / this.testCases.length;
       //   const channel = grade == 1 ? DOMConsole.INFO : DOMConsole.ERR;
-      //   outerRef.writeToConsole(channel, StringTypes.MESSAGE, "test_suite_grade", (grade * 100).toFixed(2));
+      //   this.writeToConsole(channel, StringTypes.MESSAGE, "test_suite_grade", (grade * 100).toFixed(2));
       //   return Promise.resolve(grade)
       // }).catch(err => {
-      //   outerRef.domConsole.err("Erro inesperado durante o cálculo da nota.");// try and show error messages through domconsole
-      //   outerRef.domConsole.err(err.message);
+      //   this.domConsole.err("Erro inesperado durante o cálculo da nota.");// try and show error messages through domconsole
+      //   this.domConsole.err(err.message);
       //   return Promise.resolve(0);
       // });
     } catch (error) {
-      outerRef.domConsole.err("Erro inesperado durante a execução do programa");// try and show error messages through domconsole
-      outerRef.domConsole.err(error.message);
+      this.showErrorMessage(DOMConsole.ERR, StringTypes.MESSAGE, "unexpected_execution_error");
+      this.domConsole.err(error.message);
       return Promise.resolve(0);
+    } finally {
+      Config.setConfig(this.old_config);
     }
   }
 

js/iassign-integration-functions.js

@@ -161,7 +161,11 @@ function getiLMContent () {
 function prepareActivityToEdit (ilm_cont) {
   //var content = JSON.parse(ilm_cont.split('\n::algorithm::')[0]);
   // Ver arquivo js/util/iassignHelpers.js
-  var content = ivprogCore.prepareActivityToStudentHelper(ilm_cont);
+  var content = ivprogCore.prepareActivityToStudentHelper(ilm_cont).getOrElse(null);
+  if(!content) {
+    showInvalidData();
+    return;
+  }
   var testCases = ivprogCore.getTestCases();
 
   settingsProgrammingTypes = content.settingsProgrammingType;
@@ -244,7 +248,11 @@ function includePreviousAlgorithm () {
 
 function prepareActivityToStudent (ilm_cont) {
     // Ver arquivo js/util/iassignHelpers.js
-    var content = ivprogCore.prepareActivityToStudentHelper(ilm_cont);
+    var content = ivprogCore.prepareActivityToStudentHelper(ilm_cont).getOrElse(null);
+    if(!content) {
+      showInvalidData();
+      return;
+    }
     // Casos de testes agora são delegados ao tratamento apropriado pela função acima
     // var testCases = content.testcases;
     settingsProgrammingTypes = content.settingsProgrammingType;
@@ -725,7 +733,11 @@ function full_screen() {
 function teacherAutoEval (data) {
   $.get(iLMparameters.iLM_PARAM_TeacherAutoEval, function (originalData) {
     // Ver arquivo js/util/iassignHelpers.js
-    var content = ivprogCore.prepareActivityToStudentHelper(data);
+    var content = ivprogCore.prepareActivityToStudentHelper(data).getOrElse(null);
+    if(!content) {
+      showInvalidData();
+      return;
+    }
     // Casos de testes agora são delegados ao tratamento apropriado pela função acima
     // var testCases = content.testcases;
     settingsProgrammingTypes = content.settingsProgrammingType; 
@@ -746,4 +758,15 @@ function teacherAutoEval (data) {
       blockAllEditingOptions();
     }
   });
+}
+
+function showInvalidData () {
+  $('.ui.height_100.add_accordion').dimmer({
+    closable: false
+  });
+  $('.dimmer_content_message h3').html(LocalizedStrings.getUI('text_message_error_activity_file'));
+  $('.dimmer_content_message button').text(LocalizedStrings.getUI('text_message_error_activity_reload'));
+  $('.dimmer_content_message').css('display', 'block');
+  $('.ui.height_100.add_accordion').dimmer('add content', '.dimmer_content_message');
+  $('.ui.height_100.add_accordion').dimmer('show');
 }

js/io/domConsole.js

@@ -194,7 +194,7 @@ export class DOMConsole {
       textDiv.innerHTML = this.getOutputText(text);
       this.termDiv.insertBefore(textDiv, this.inputDiv);
       this.scrollTerm();
-    }, 50);
+    }, 5);
     this.pending_writes.push(pending_write);
   }
 
@@ -211,7 +211,7 @@ export class DOMConsole {
       textDiv.classList.add(divClass);
       this.termDiv.insertBefore(textDiv, this.inputDiv);
       this.scrollTerm();
-    }, 50);
+    }, 5);
     this.pending_writes.push(pending_write);
   }
 

js/processor/error/processorErrorFactory.js

@@ -102,6 +102,9 @@ export const ProcessorErrorFactory  = Object.freeze({
     const context = [exp];
     return createSemanticError("loop_condition_type", context);
   },
+  /**
+   * @deprecated 01/10/2019
+   */
   endless_loop_full: (sourceInfo) => {
     if(sourceInfo) {
       const context = [sourceInfo.line];
@@ -493,6 +496,9 @@ export const ProcessorErrorFactory  = Object.freeze({
   negative_sqrt_value: (source_info) => {
     return createRuntimeError("negative_sqrt_value",[source_info.line]);
   },
+  /**
+   * @deprecated 01/10/2019
+   */
   exceeded_recursive_calls: (source_info) => {
     const context = [source_info.line];
     return createRuntimeError("exceeded_recursive_calls", context);
@@ -512,5 +518,8 @@ export const ProcessorErrorFactory  = Object.freeze({
   invalid_for_pass: (exp, source_info) => {
     const context = [source_info.line, exp];
     return createSemanticError("invalid_for_pass", context);
+  },
+  exceed_max_instructions: () => {
+    return createRuntimeError('exceed_max_instructions');
   }
 });

js/processor/ivprogProcessor.js

@@ -22,14 +22,6 @@ import { LocalizedStrings } from '../services/localizedStringsService';
 
 export class IVProgProcessor {
 
-  static get LOOP_TIMEOUT () {
-    return Config.loopTimeout;
-  }
-
-  static set LOOP_TIMEOUT (ms) {
-    Config.setConfig({loopTimeout: ms});
-  }
-
   static get MAIN_INTERNAL_ID () {
     return "$main";
   }
@@ -41,14 +33,14 @@ export class IVProgProcessor {
     this.context = [Context.BASE];
     this.input = null;
     this.forceKill = false;
-    this.loopTimers = [];
     this.output = null;
     this.mode = Modes.RUN;
     /**
      * Stores the sourceInfo of every function call, command or expression
      */
     this.function_call_stack = [];
-    this.command_count = 0;
+    this.instruction_count = 0;
+    this.function_call_count = 0;
   }
 
   registerInput (input) {
@@ -89,7 +81,8 @@ export class IVProgProcessor {
     this.globalStore = new Store("$global");
     this.stores = [this.globalStore];
     this.context = [Context.BASE];
-    this.command_count = 0;
+    this.instruction_count = 0;
+    this.mode = Modes.RUN;
   }
 
   interpretAST () {
@@ -138,17 +131,24 @@ export class IVProgProcessor {
     const funcName = func.isMain ? IVProgProcessor.MAIN_INTERNAL_ID : func.name;
     const funcStore = new Store(funcName);
     funcStore.extendStore(this.globalStore);
-    const newFuncStore$ = this.associateParameters(func.formalParameters, actualParameters, store, funcStore);
-    return newFuncStore$.then(sto => {
-      this.context.push(Context.FUNCTION);
-      this.stores.push(sto);
-      return this.executeCommands(sto, func.variablesDeclarations)
-        .then(stoWithVars => this.executeCommands(stoWithVars, func.commands)).then(finalSto => {
-          this.stores.pop();
-          this.context.pop();
-          return finalSto;
-        });
+    return new Promise((resolve, reject) => {
+      const run_lambda = () => {
+        const newFuncStore$ = this.associateParameters(func.formalParameters, actualParameters, store, funcStore);
+        newFuncStore$.then(sto => {
+          this.context.push(Context.FUNCTION);
+          this.stores.push(sto);
+          return this.executeCommands(sto, func.variablesDeclarations)
+            .then(stoWithVars => this.executeCommands(stoWithVars, func.commands)).then(finalSto => {
+              this.stores.pop();
+              this.context.pop();
+              return finalSto;
+            });
+        }).then(resolve)
+        .catch(reject);
+      }
+      run_lambda();
     });
+    
   }
 
   associateParameters (formal_params, effective_params, caller_store, callee_store) {
@@ -224,10 +224,12 @@ export class IVProgProcessor {
   }
 
   executeCommand (store, cmd) {
-    this.command_count += 1;
+    this.instruction_count += 1;
     return new Promise((resolve, reject) => {
       const command_lambda = () => {
-        if(this.forceKill) {
+        if(this.instruction_count >= Config.max_instruction_count) {
+          return reject(ProcessorErrorFactory.exceed_max_instructions());
+        } else if(this.forceKill) {
           return reject("FORCED_KILL!");
         } else if (store.mode === Modes.PAUSE) {
           return resolve(this.executeCommand(store, cmd));
@@ -266,14 +268,13 @@ export class IVProgProcessor {
           return reject(ProcessorErrorFactory.unknown_command(cmd.sourceInfo))
         }
       };
-      if(this.command_count % 100 == 0) {
+      if(this.instruction_count % Config.suspend_threshold == 0) {
         //every 100th command should briefly delay its execution in order to allow the browser to process other things
         setTimeout(command_lambda, 5);
       } else {
         command_lambda();
       }
     })
-    
   }
 
   executeSysCall (store, cmd) {
@@ -288,9 +289,9 @@ export class IVProgProcessor {
     } else {
       func = this.findFunction(cmd.id);
     }
-    if(this.function_call_stack.length >= Config.max_call_stack) {
-      return Promise.reject(ProcessorErrorFactory.exceeded_recursive_calls(cmd.sourceInfo));
-    }
+    // if(this.function_call_stack.length >= Config.max_call_stack) {
+    //   return Promise.reject(ProcessorErrorFactory.exceeded_recursive_calls(cmd.sourceInfo));
+    // }
     this.function_call_stack.push(cmd.sourceInfo);
     return this.runFunction(func, cmd.actualParameters, store)
       .then(sto => {
@@ -382,14 +383,12 @@ export class IVProgProcessor {
 
   executeRepeatUntil (store, cmd) {
     try {
-      this.loopTimers.push(Date.now());
       this.context.push(Context.BREAKABLE);
       const $newStore = this.executeCommands(store, cmd.commands);
       return $newStore.then(sto => {
         if(sto.mode === Modes.BREAK) {
           this.context.pop();
           sto.mode = Modes.RUN;
-          this.loopTimers.pop();
           return sto;
         }
         const $value = this.evaluateExpression(sto, cmd.expression);
@@ -399,17 +398,9 @@ export class IVProgProcessor {
           }
           if (!vl.get()) {
             this.context.pop();
-            if(this.loopTimers.length > 0) {
-              const time = this.loopTimers[0];
-              if(Date.now() - time >= IVProgProcessor.LOOP_TIMEOUT) {
-                this.forceKill = true;
-                return Promise.reject(ProcessorErrorFactory.endless_loop_full(cmd.sourceInfo));
-              }
-            }
             return this.executeCommand(sto, cmd);
           } else {
             this.context.pop();
-            this.loopTimers.pop();
             return sto;
           }
         })
@@ -421,7 +412,6 @@ export class IVProgProcessor {
 
   executeWhile (store, cmd) {
     try {
-      this.loopTimers.push(Date.now());
       this.context.push(Context.BREAKABLE);
       const $value = this.evaluateExpression(store, cmd.expression);
       return $value.then(vl => {
@@ -431,22 +421,13 @@ export class IVProgProcessor {
             return $newStore.then(sto => {
               this.context.pop();
               if (sto.mode === Modes.BREAK) {
-                this.loopTimers.pop();
                 sto.mode = Modes.RUN;
                 return sto;
               }
-              if (this.loopTimers.length > 0) {
-                const time = this.loopTimers[0];
-                if(Date.now() - time >= IVProgProcessor.LOOP_TIMEOUT) {
-                  this.forceKill = true;
-                  return Promise.reject(ProcessorErrorFactory.endless_loop_full(cmd.sourceInfo));
-                }
-              }
               return this.executeCommand(sto, cmd);
             });
           } else {
             this.context.pop();
-            this.loopTimers.pop();
             return store;
           }
         } else {
@@ -488,7 +469,7 @@ export class IVProgProcessor {
     try {
       const funcName = store.name === IVProgProcessor.MAIN_INTERNAL_ID ? 
         LanguageDefinedFunction.getMainFunctionName() : store.name;
-      console.log(funcName,  store.name === IVProgProcessor.MAIN_INTERNAL_ID);
+      // console.log(funcName,  store.name === IVProgProcessor.MAIN_INTERNAL_ID);
       const func = this.findFunction(store.name);
       const funcType = func.returnType;
       const $value = this.evaluateExpression(store, cmd.expression);
@@ -732,28 +713,46 @@ export class IVProgProcessor {
   }
 
   evaluateExpression (store, exp) {
-    if (exp instanceof Expressions.UnaryApp) {
-      return this.evaluateUnaryApp(store, exp);
-    } else if (exp instanceof Expressions.InfixApp) {
-      return this.evaluateInfixApp(store, exp);
-    } else if (exp instanceof Expressions.ArrayAccess) {
-      return this.evaluateArrayAccess(store, exp);
-    } else if (exp instanceof Expressions.VariableLiteral) {
-      return this.evaluateVariableLiteral(store, exp);
-    } else if (exp instanceof Expressions.IntLiteral) {
-      return this.evaluateLiteral(store, exp);
-    } else if (exp instanceof Expressions.RealLiteral) {
-      return this.evaluateLiteral(store, exp);
-    } else if (exp instanceof Expressions.BoolLiteral) {
-      return this.evaluateLiteral(store, exp);
-    } else if (exp instanceof Expressions.StringLiteral) {
-      return this.evaluateLiteral(store, exp);
-    } else if (exp instanceof Expressions.ArrayLiteral) {
-      return Promise.reject(new Error("Internal Error: The system should not eval an array literal."))
-    } else if (exp instanceof Expressions.FunctionCall) {
-      return this.evaluateFunctionCall(store, exp);
-    }
-    return Promise.resolve(null);
+    this.instruction_count += 1;
+    return new Promise((resolve, reject) => {
+      const expression_lambda = () => {
+        if (this.mode === Modes.ABORT) {
+          return reject(LocalizedStrings.getMessage('aborted_execution'));
+        }
+        if(this.instruction_count >= Config.max_instruction_count) {
+          return reject(new Error("Número de instruções excedeu o limite definido. Verifique se seu código não possui laços infinitos ou muitas chamadas de funções recursivas."))
+        }
+        if (exp instanceof Expressions.UnaryApp) {
+          return resolve(this.evaluateUnaryApp(store, exp));
+        } else if (exp instanceof Expressions.InfixApp) {
+          return resolve(this.evaluateInfixApp(store, exp));
+        } else if (exp instanceof Expressions.ArrayAccess) {
+          return resolve(this.evaluateArrayAccess(store, exp));
+        } else if (exp instanceof Expressions.VariableLiteral) {
+          return resolve(this.evaluateVariableLiteral(store, exp));
+        } else if (exp instanceof Expressions.IntLiteral) {
+          return resolve(this.evaluateLiteral(store, exp));
+        } else if (exp instanceof Expressions.RealLiteral) {
+          return resolve(this.evaluateLiteral(store, exp));
+        } else if (exp instanceof Expressions.BoolLiteral) {
+          return resolve(this.evaluateLiteral(store, exp));
+        } else if (exp instanceof Expressions.StringLiteral) {
+          return resolve(this.evaluateLiteral(store, exp));
+        } else if (exp instanceof Expressions.ArrayLiteral) {
+          return reject(new Error("Internal Error: The system should not eval an array literal."))
+        } else if (exp instanceof Expressions.FunctionCall) {
+          return resolve(this.evaluateFunctionCall(store, exp));
+        }
+        return resolve(null);
+      };
+      if(this.instruction_count % Config.suspend_threshold == 0) {
+        //every 100th command should briefly delay its execution in order to allow the browser to process other things
+        setTimeout(expression_lambda, 5);
+      } else {
+        expression_lambda();
+      }
+    });
+    
   }
 
   evaluateFunctionCall (store, exp) {

js/processor/lib/io.js

@@ -56,7 +56,6 @@ export function createInputFun () {
         const error = ProcessorErrorFactory.invalid_read_type(text, stringInfo.type, stringInfo.dim, realObject.getRefObj(), this.function_call_stack.pop());
         return Promise.reject(error);
       }
-      this.loopTimers.splice(0, this.loopTimers.length)
       const stoValue = new StoreValue(type, result);
       store.updateStore('p1', stoValue);
       store.mode = Modes.RETURN;

js/util/config.js

@@ -1,13 +1,14 @@
 class ConfigObject {
 
   constructor () {
-    this.loopTimeout = Number.MAX_SAFE_INTEGER;
     this.decimalPlaces = 8;
     this.intConvertRoundMode = 2;
     this.default_lang = 'pt';
     this.enable_type_casting = true;
     this.idle_input_interval = 5000;
-    this.max_call_stack = 10000;
+    this.suspend_threshold = 100;
+    // this.max_instruction_count = 350250; - automated evaluation limit
+    this.max_instruction_count = Number.MAX_SAFE_INTEGER;
   }
 
   setConfig (opts) {

js/util/iassignHelpers.js

@@ -3,7 +3,8 @@ import { generate } from "../visualUI/code_generator";
 import { IVProgAssessment } from "../assessment/ivprogAssessment";
 import { TestConsole } from "./testConsole";
 import { parseLogs } from "./../services/userLog";
-import { LocalizedStrings } from './../services/localizedStringsService';
+import { SemanticAnalyser } from "../processor/semantic/semanticAnalyser";
+import { Maybe } from "./maybe";
 
 function parseActivityData (data) {
   let algorithm_in_ilm = null;
@@ -19,56 +20,45 @@ function parseActivityData (data) {
     content = JSON.parse(data.split('\n::algorithm::')[0]);
     content['algorithm_in_ilm'] = algorithm_in_ilm;
   } catch (e) {
-    $('.ui.height_100.add_accordion').dimmer({
-      closable: false
-    });
-    $('.dimmer_content_message h3').html(LocalizedStrings.getUI('text_message_error_activity_file'));
-    $('.dimmer_content_message button').text(LocalizedStrings.getUI('text_message_error_activity_reload'));
-    $('.dimmer_content_message').css('display', 'block');
-    $('.ui.height_100.add_accordion').dimmer('add content', '.dimmer_content_message');
-    $('.ui.height_100.add_accordion').dimmer('show');
     console.error(e);
+    return Maybe.none();
   }
-  return content;
+  return Maybe.some(content);
 }
 
 export function prepareActivityToStudentHelper (ilm_cont) {
-  const content = parseActivityData(ilm_cont);
-  const testCases = content.testcases;
-  setTestCases(testCases);
+  const maybe_content = parseActivityData(ilm_cont);
+  return maybe_content.map(content => {
+    const testCases = content.testcases;
+    setTestCases(testCases);
 
-  let prog_type = null;
-  if (content.settings_programming_type) {
-    prog_type = content.settings_programming_type[0].value;
-  } else {
-    prog_type = "visual";
-  }
+    let prog_type = null;
+    if (content.settings_programming_type) {
+      prog_type = content.settings_programming_type[0].value;
+    } else {
+      prog_type = "visual";
+    }
 
-  return {
-    settingsProgrammingType: prog_type,
-    settingsDataTypes: content.settings_data_types,
-    settingsCommands: content.settings_commands,
-    settingsFunctions: content.settings_functions,
-    algorithmInIlm: content.algorithm_in_ilm,
-    settingsFilter: content.settings_filter
-  }
+    return {
+      settingsProgrammingType: prog_type,
+      settingsDataTypes: content.settings_data_types,
+      settingsCommands: content.settings_commands,
+      settingsFunctions: content.settings_functions,
+      algorithmInIlm: content.algorithm_in_ilm,
+      settingsFilter: content.settings_filter
+    }
+  });
 }
 
-export function autoEval (originalData, callback) {
-  const code = generate();
-  const original = parseActivityData(originalData);
-  if (code == null) {
-    return callback(-1);
-  } else {
-    if (!compareTestcases(original.testcases, getTestCases())) {
-      return callback(-2);
+function compareArray (a, b) {
+  for (let i = 0; i < a.length; ++i) {
+    const elementA = a[i];
+    const elementB = b[i];
+    if (elementA != elementB) {
+      return false;
     }
-    const autoAssessment = new IVProgAssessment(code, getTestCases(), new TestConsole([]));
-    autoAssessment.runTest().then( grade => callback(grade)).catch(err => {
-      console.log(err);
-      callback(0);
-    });
   }
+  return true;
 }
 
 function compareTestcases (original, student) {
@@ -87,12 +77,24 @@ function compareTestcases (original, student) {
   }
 }
 
-function compareArray (a, b) {
-  for (let i = 0; i < a.length; ++i) {
-    const elementA = a[i];
-    const elementB = b[i];
-    if (elementA != elementB) {
-      return false;
+export function autoEval (originalData, callback) {
+  const code = generate();
+  const original = parseActivityData(originalData).getOrElse(undefined);
+  if(original == null) {
+    alert("iAssign did not provide the original activity data!");
+    return callback(null);
+  }
+  if (code == null) {
+    return callback(-1);
+  } else {
+    if (!compareTestcases(original.testcases, getTestCases())) {
+      return callback(-2);
     }
+    const ast_code = SemanticAnalyser.analyseFromSource(code);
+    const autoAssessment = new IVProgAssessment(ast_code, getTestCases(), new TestConsole([]));
+    autoAssessment.runTest().then( grade => callback(grade)).catch(err => {
+      console.log(err);
+      callback(0);
+    });
   }
-}
+}