CI: Improve mark validation

test/validate_ci_marklog.py

@@ -11,13 +11,32 @@ def read_marklog():
 
 
 def check_marklog(lines, expected_values):
-    for line in lines:
+    try:
-        # Remove the newline character
+        # Check to make sure the marklog contains all the expected values and nothing else
-        line = line.strip()
+        found_values = []
-        if line not in expected_values:
+        for line in lines:
-            print("Line not found in marklog: " + line)
+            # Remove the newline character
-            return False
+            line = line.strip()
-    return True
+            if line not in expected_values:
+                raise Exception("Line not found in marklog: " + line)
+
+
+            found_values.append(line)
+
+        # Check to make sure the marklog contains the same number of values as the expected values
+        if len(found_values) != len(expected_values):
+            raise Exception("Marklog did not contain the same number of values as the expected values, found " + 
+                str(len(found_values)) + " values, expected " + str(len(expected_values)) + " values")
+        
+        # Check that the two lists contain the same values
+        if sorted(found_values) != sorted(expected_values):
+            raise Exception("Marklog did not contain the same values as the expected values, found:\n" +
+                "\n".join(sorted(found_values)) + "\n\nExpected:\n" + "\n".join(sorted(expected_values)))
+
+        return True
+    except Exception as e:
+        print(e)
+        return False
         
 
 def main():
@@ -35,10 +54,12 @@ def main():
 
     lines = read_marklog()
     if not check_marklog(lines, expected_values):
-        print("Marklog did not contain the expected values")
+        print("Failed to validate marklog")
         exit(1)
-    else:
+
-        print("Marklog contained the expected values")
+    print("Successfully validated marklog")
+    exit(0)
+
 
 
 if __name__ == "__main__":