[stats] utility to diff two stats directories

Summary:
Being able to compare stats across runs is handy. Add options to
`infer-reportdiff` to do so.

Reviewed By: skcho

Differential Revision:
D55426558

Privacy Context Container: L1208441

fbshipit-source-id: 73eb680882aaba9b78ba1d51acebefdca134cf2c

infer/man/man1/infer-full.txt

@@ -2140,6 +2140,17 @@ OPTIONS
  (Conversely: --no-starvation-only)
  See also infer-analyze(1).
 
+ --stats-dir-current path
+ The infer-out/stats from the current run. Together with
+ --stats-dir-previous, make infer reportdiff compute the difference
+ between two stats directories and output the results in
+ infer-out/differential/stats_*.json files.
+ See also infer-reportdiff(1).
+
+ --stats-dir-previous path
+ The infer-out/stats from a previous run. See --stats-dir-current.
+ See also infer-reportdiff(1).
+
  --store-analysis-schedule
  Activates: Store the analysis schedule for later replay, honoring
  --replay-analysis-schedule-file if present. This can be useful to
@@ -3266,6 +3277,12 @@ INTERNAL OPTIONS
  Activates: Run whole-program starvation analysis (Conversely:
  --no-starvation-whole-program)
 
+ --stats-dir-current-reset
+ Cancel the effect of --stats-dir-current.
+
+ --stats-dir-previous-reset
+ Cancel the effect of --stats-dir-previous.
+
  --no-subtype-multirange
  Deactivates: Use the multirange subtyping domain. Used in the Java
  frontend and in biabduction. (Conversely: --subtype-multirange)

infer/man/man1/infer-reportdiff.txt

@@ -85,6 +85,15 @@ OPTIONS
  computing differential reports (Conversely:
  --skip-duplicated-types)
 
+ --stats-dir-current path
+ The infer-out/stats from the current run. Together with
+ --stats-dir-previous, make infer reportdiff compute the difference
+ between two stats directories and output the results in
+ infer-out/differential/stats_*.json files.
+
+ --stats-dir-previous path
+ The infer-out/stats from a previous run. See --stats-dir-current.
+
 ENVIRONMENT
  INFER_ARGS, INFERCONFIG, INFER_STRICT_MODE
  See the ENVIRONMENT section in the manual of infer(1).

infer/man/man1/infer.txt

@@ -2140,6 +2140,17 @@ OPTIONS
  (Conversely: --no-starvation-only)
  See also infer-analyze(1).
 
+ --stats-dir-current path
+ The infer-out/stats from the current run. Together with
+ --stats-dir-previous, make infer reportdiff compute the difference
+ between two stats directories and output the results in
+ infer-out/differential/stats_*.json files.
+ See also infer-reportdiff(1).
+
+ --stats-dir-previous path
+ The infer-out/stats from a previous run. See --stats-dir-current.
+ See also infer-reportdiff(1).
+
  --store-analysis-schedule
  Activates: Store the analysis schedule for later replay, honoring
  --replay-analysis-schedule-file if present. This can be useful to

infer/src/base/Config.ml

@@ -3403,6 +3403,20 @@ and starvation_whole_program =
  "Run whole-program starvation analysis"
 
 
+and stats_dir_current =
+ CLOpt.mk_path_opt ~long:"stats-dir-current"
+ ~in_help:InferCommand.[(ReportDiff, manual_generic)]
+ "The infer-out/stats from the current run. Together with $(b,--stats-dir-previous), make \
+ $(i,infer reportdiff) compute the difference between two stats directories and output the \
+ results in infer-out/differential/stats_*.json files."
+
+
+and stats_dir_previous =
+ CLOpt.mk_path_opt ~long:"stats-dir-previous"
+ ~in_help:InferCommand.[(ReportDiff, manual_generic)]
+ "The infer-out/stats from a previous run. See $(b,--stats-dir-current)."
+
+
 and store_analysis_schedule =
  CLOpt.mk_bool ~long:"store-analysis-schedule"
  ~in_help:InferCommand.[(Analyze, manual_scheduler)]
@@ -4673,6 +4687,10 @@ and starvation_strict_mode = !starvation_strict_mode
 
 and starvation_whole_program = !starvation_whole_program
 
+and stats_dir_current = !stats_dir_current
+
+and stats_dir_previous = !stats_dir_previous
+
 and store_analysis_schedule = !store_analysis_schedule
 
 and subtype_multirange = !subtype_multirange

infer/src/base/Config.mli

@@ -846,6 +846,10 @@ val starvation_strict_mode : bool
 
 val starvation_whole_program : bool
 
+val stats_dir_current : string option
+
+val stats_dir_previous : string option
+
 val store_analysis_schedule : bool
 
 val subtype_multirange : bool

infer/src/base/ScubaLogging.ml

@@ -75,7 +75,7 @@ let log_to_debug samples =
  Utils.with_file_out ~append:true log_file ~f:(fun out ->
  List.iter samples ~f:(fun sample ->
  Yojson.to_channel out (Scuba.sample_to_json sample) ;
- Printf.fprintf out ",\n" ) )
+ Printf.fprintf out "\n" ) )
 
 
 (** Consider buffering or batching if proves to be a problem *)

infer/src/integration/InferCommandImplementation.ml

@@ -301,23 +301,27 @@ let report () =
 
 
 let report_diff () =
- (* at least one report must be passed in input to compute differential *)
+ (* at least one pair of reports must be passed as input to compute a differential *)
+ let open Config in
  match
- Config.
- ( report_current
- , report_previous
- , costs_current
- , costs_previous
- , config_impact_current
- , config_impact_previous )
+ ( Option.both report_current report_previous
+ , Option.both costs_current costs_previous
+ , Option.both config_impact_current config_impact_previous
+ , Option.both stats_dir_current stats_dir_previous )
  with
- | None, None, None, None, None, None ->
+ | None, None, None, None ->
  L.die UserError
- "Expected at least one argument among '--report-current', '--report-previous', \
- '--costs-current', '--costs-previous', '--config-impact-current', and \
- '--config-impact-previous'\n"
+ "Expected at least one pair of arguments among '--report-current'/'--report-previous', \
+ '--costs-current'/'--costs-previous', \
+ '--config-impact-current'/'--config-impact-previous', or \
+ '--stats-dir-current'/'--stats-dir-previous'"
  | _ ->
- ReportDiff.reportdiff ~current_report:Config.report_current
- ~previous_report:Config.report_previous ~current_costs:Config.costs_current
- ~previous_costs:Config.costs_previous ~current_config_impact:Config.config_impact_current
- ~previous_config_impact:Config.config_impact_previous
+ if
+ (is_some @@ Option.both report_current report_previous)
+ || (is_some @@ Option.both costs_current costs_previous)
+ || (is_some @@ Option.both config_impact_current config_impact_previous)
+ then
+ ReportDiff.reportdiff ~report_current ~report_previous ~costs_current ~costs_previous
+ ~config_impact_current ~config_impact_previous ;
+ Option.both stats_dir_previous stats_dir_current
+ |> Option.iter ~f:(fun (previous, current) -> StatsDiff.diff ~previous ~current)

infer/src/integration/ReportDiff.ml

@@ -4,12 +4,13 @@
  * This source code is licensed under the MIT license found in the
  * LICENSE file in the root directory of this source tree.
  *)
+
 open! IStd
 
-let reportdiff ~current_report:current_report_fname ~previous_report:previous_report_fname
- ~current_costs:current_costs_fname ~previous_costs:previous_costs_fname
- ~current_config_impact:current_config_impact_fname
- ~previous_config_impact:previous_config_impact_fname =
+let reportdiff ~report_current:current_report_fname ~report_previous:previous_report_fname
+ ~costs_current:current_costs_fname ~costs_previous:previous_costs_fname
+ ~config_impact_current:current_config_impact_fname
+ ~config_impact_previous:previous_config_impact_fname =
  let load_aux ~f filename_opt =
  Option.value_map
  ~f:(fun filename -> Atdgen_runtime.Util.Json.from_file f filename)

infer/src/integration/ReportDiff.mli

@@ -8,10 +8,10 @@
 open! IStd
 
 val reportdiff :
- current_report:string option
- -> previous_report:string option
- -> current_costs:string option
- -> previous_costs:string option
- -> current_config_impact:string option
- -> previous_config_impact:string option
+ report_current:string option
+ -> report_previous:string option
+ -> costs_current:string option
+ -> costs_previous:string option
+ -> config_impact_current:string option
+ -> config_impact_previous:string option
  -> unit

infer/src/integration/StatsDiff.ml

@@ -0,0 +1,148 @@
+(*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *)
+
+open! IStd
+module L = Logging
+
+(**
+
+ entries are of the form
+
+
+ {
+ "int": {
+ "is_main_process": 1,
+ "pid": 1234,
+ "time": 420844200,
+ "value": 123456
+ },
+ "normal": {
+ "command": "run",
+ "event": "time.dbwriter.useful_time_user",
+ "hostname": "darkstar",
+ "infer_commit": "deadbeef"
+ },
+ "tags": {}
+ }
+ *)
+
+let error ~expected json =
+ L.die InternalError "when parsing json: expected %s but got '%a'" expected Yojson.Safe.pp json
+
+
+let get_field field_name json =
+ match json with
+ | `Assoc assoc ->
+ List.Assoc.find ~equal:String.equal assoc field_name
+ | _ ->
+ error ~expected:"record" json
+
+
+let get_field_exn field_name json =
+ match get_field field_name json with
+ | Some json' ->
+ json'
+ | None ->
+ error ~expected:(Printf.sprintf "field '%s'" field_name) json
+
+
+let get_string_exn json = match json with `String s -> s | json -> error ~expected:"string" json
+
+let get_event_exn json = json |> get_field_exn "normal" |> get_field_exn "event" |> get_string_exn
+
+let get_value_exn json =
+ (* some values are strings ("normal"), some are ints, none are neither *)
+ match json |> get_field_exn "normal" |> get_field "message" with
+ | Some value ->
+ value
+ | None ->
+ json |> get_field_exn "int" |> get_field_exn "value"
+
+
+let collate_stats_in_dir stats_dir =
+ Iter.fold
+ (fun json_rows dir_entry ->
+ if Filename.check_suffix dir_entry ".jsonl" then
+ Iter.fold
+ (fun json_rows line -> Yojson.Safe.from_string line :: json_rows)
+ json_rows
+ (Iter.from_labelled_iter @@ In_channel.iter_lines @@ In_channel.create dir_entry)
+ else json_rows )
+ []
+ (Iter.from_labelled_iter @@ Utils.iter_dir stats_dir)
+
+
+let diff_values entry1 entry2 =
+ let v1 = get_value_exn entry1 in
+ let v2 = get_value_exn entry2 in
+ if Yojson.Safe.equal v1 v2 then `Same v1 else `Diff (v1, v2)
+
+
+let compute_diff ~before ~after =
+ let sort_entries entries =
+ List.sort
+ ~compare:(fun json1 json2 -> String.compare (get_event_exn json1) (get_event_exn json2))
+ entries
+ in
+ let before = sort_entries before in
+ let after = sort_entries after in
+ let rec diff_aux ~extra_before ~extra_after ~unchanged ~diff before after =
+ match (before, after) with
+ | [], _ ->
+ (extra_before, after @ extra_after, unchanged, diff)
+ | _, [] ->
+ (before @ extra_before, extra_after, unchanged, diff)
+ | entry1 :: before', entry2 :: after' ->
+ let event1 = get_event_exn entry1 in
+ let event2 = get_event_exn entry2 in
+ let cmp = String.compare event1 event2 in
+ if Int.equal cmp 0 then
+ (* [event1 = event2] *)
+ match diff_values entry1 entry2 with
+ | `Same v ->
+ diff_aux ~extra_before ~extra_after ~unchanged:((event1, v) :: unchanged) ~diff
+ before' after'
+ | `Diff d ->
+ diff_aux ~extra_before ~extra_after ~unchanged ~diff:((event1, d) :: diff) before'
+ after'
+ else
+ let extra_before, extra_after, before'', after'' =
+ if cmp < 0 then
+ ((* [event1 < event2] *)
+ entry1 :: extra_before, extra_after, before', after)
+ else ((* [event1 > event2] *) extra_before, entry2 :: extra_after, before, after')
+ in
+ diff_aux ~extra_before ~extra_after ~unchanged ~diff before'' after''
+ in
+ let extra_before, extra_after, unchanged, diff =
+ diff_aux ~extra_before:[] ~extra_after:[] ~unchanged:[] ~diff:[] before after
+ in
+ ( `List extra_before
+ , `List extra_after
+ , `List
+ (List.map unchanged ~f:(fun (event, value) ->
+ `Assoc [("event", `String event); ("value", value)] ) )
+ , `List
+ (List.map diff ~f:(fun (event, (before, after)) ->
+ `Assoc [("event", `String event); ("value_before", before); ("value_after", after)] ) )
+ )
+
+
+let output_diff (extra_before, extra_after, unchanged, diff) =
+ let out_dir = ResultsDir.get_path Differential in
+ Unix.mkdir_p out_dir ;
+ Yojson.Safe.to_file (out_dir ^/ "stats_previous_only.json") extra_before ;
+ Yojson.Safe.to_file (out_dir ^/ "stats_current_only.json") extra_after ;
+ Yojson.Safe.to_file (out_dir ^/ "stats_unchanged.json") unchanged ;
+ Yojson.Safe.to_file (out_dir ^/ "stats_diff.json") diff ;
+ ()
+
+
+let diff ~previous:stats_dir_previous ~current:stats_dir_current =
+ let stats_previous = collate_stats_in_dir stats_dir_previous in
+ let stats_current = collate_stats_in_dir stats_dir_current in
+ compute_diff ~before:stats_previous ~after:stats_current |> output_diff

infer/src/integration/StatsDiff.mli

@@ -0,0 +1,10 @@
+(*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *)
+
+open! IStd
+
+val diff : previous:string -> current:string -> unit