Implementation of type_3_clone_detection

.gitignore

@@ -12,8 +12,9 @@ data/
 ### Scyros ###
 ghtokens.csv
 !tests/data
-
+outputs/
+src/phases/alt_parse.rs //Got broken by refactoring but is not in use currently, so ignoring for now
 *.zip
 *.tar.gz
-
-result/
+result/
+examples/

Cargo.toml

@@ -34,6 +34,7 @@ crossbeam="0.7"
 crossbeam-channel="0.5.0"
 csv="1.1"
 curl="0.4"
+either = "1.15.0"
 indicatif = "0.17.9"
 json="0.12"
 lazy_static = "1.4.0"

src/bin/main.rs

@@ -16,7 +16,8 @@ use anyhow::{anyhow, Context, Result};
 use clap::{Arg, ArgAction, Command};
 use scyros::phases::{
     download, duplicate_files, duplicate_ids, extract_benchmarks, filter_languages,
-    filter_metadata, forks, ids, languages, metadata, parse, pull_request,
+    filter_metadata, forks, ids, languages, metadata, parse, pull_request, tokenizer,
+    type_3_duplicate_files,
 };
 use scyros::utils::logger::Logger;
 use tracing::{error, info};
@@ -37,6 +38,8 @@ fn cli() -> Command {
         .subcommand(duplicate_files::cli())
         .subcommand(parse::cli())
         .subcommand(extract_benchmarks::cli())
+        .subcommand(tokenizer::cli())
+        .subcommand(type_3_duplicate_files::cli())
         .arg(
             Arg::new("debug")
                 .long("debug")
@@ -234,6 +237,49 @@ fn main() {
                                     &logger,
                                 )
                             }
+                            /* else if subcommand == alt_parse::cli().get_name() {
+                                alt_parse::run(
+                                    cli_subargs.get_one::<String>("input").unwrap(),
+                                    cli_subargs.get_one::<String>("output").map(|x| x.as_str()),
+                                    cli_subargs.get_one::<String>("logs").map(|x| x.as_str()),
+                                    cli_subargs
+                                        .get_many::<String>("lang")
+                                        .map(|v|
+                                        v.map(|s| s.as_str())
+                                        .collect::<Vec<&str>>()),
+                                    cli_subargs.get_one::<String>("failures").unwrap(),
+                                    *cli_subargs.get_one::<usize>("threads").unwrap(),
+                                    *cli_subargs.get_one::<u64>("seed").unwrap(),
+                                    cli_subargs.get_flag("force"),
+                                    &mut logger,
+                                )
+                            } */
+                            else if subcommand == tokenizer::cli().get_name() {
+                                tokenizer::run(
+                                    cli_subargs.get_one::<String>("input").unwrap(),
+                                    //cli_subargs.get_one::<String>("output").map(|x| x.as_str()),
+                                    //cli_subargs.get_one::<String>("language").unwrap(),
+                                    cli_subargs.get_one::<String>("example_word").unwrap(),
+                                    &logger,
+                                )
+                            }
+                            else if subcommand == type_3_duplicate_files::cli().get_name() {
+                                type_3_duplicate_files::run(
+                                    cli_subargs.get_one::<String>("input").unwrap(),
+                                    cli_subargs.get_one::<String>("output").map(|x| x.as_str()),
+                                    cli_subargs.get_one::<String>("map").map(|x| x.as_str()),
+                                    cli_subargs.get_one::<String>("logs").map(|x| x.as_str()),
+                                    /* languages */
+                                    cli_subargs.get_one::<String>("language").map(|s| s.as_str()),
+                                    *cli_subargs.get_one::<usize>("threads").unwrap(),
+                                    *cli_subargs.get_one::<usize>("p_prefix").unwrap(),
+                                    *cli_subargs.get_one::<f64>("threshold").unwrap(),
+                                    cli_subargs.get_one::<String>("example_word"),
+                                    cli_subargs.get_flag("force"),
+                                    cli_subargs.get_one::<String>("header").unwrap(),
+                                    &logger,
+                                )
+                            }
                             else {
                                 Err(anyhow!("The subcommand {subcommand} is not available. Run the program with the --help flag to see the list of subcommands"))
                             }