From db88385fbde8ed3a9c81fd83fc60b850696c20d0 Mon Sep 17 00:00:00 2001 From: Larry Bowden Date: Sat, 20 Mar 2021 18:09:39 -0500 Subject: [PATCH 1/3] added encoding utf-8 specification --- gpt_2_simple/gpt_2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gpt_2_simple/gpt_2.py b/gpt_2_simple/gpt_2.py index 37297a2..2e9833b 100644 --- a/gpt_2_simple/gpt_2.py +++ b/gpt_2_simple/gpt_2.py @@ -494,7 +494,7 @@ def generate(sess, gen_text = trunc_text.group(1) gen_text = gen_text.lstrip('\n') if destination_path: - f.write("{}\n{}".format(gen_text, sample_delim)) + f.write("{}\n{}".format(gen_text, sample_delim), encoding='utf-8') if not return_as_list and not destination_path: print("{}\n{}".format(gen_text, sample_delim), end='') gen_texts.append(gen_text) From 43e640edc9f6da0b1fb51b6e9ef006d6bb780c49 Mon Sep 17 00:00:00 2001 From: Larry Bowden Date: Sat, 20 Mar 2021 18:31:28 -0500 Subject: [PATCH 2/3] added encoding utf-8 specification --- gpt_2_simple/gpt_2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gpt_2_simple/gpt_2.py b/gpt_2_simple/gpt_2.py index 2e9833b..1134a26 100644 --- a/gpt_2_simple/gpt_2.py +++ b/gpt_2_simple/gpt_2.py @@ -465,7 +465,7 @@ def generate(sess, )[:, 1:] if destination_path: - f = open(destination_path, 'w') + f = open(destination_path, 'w', encoding='utf-8') generated = 0 gen_texts = [] while generated < nsamples: @@ -494,7 +494,7 @@ def generate(sess, gen_text = trunc_text.group(1) gen_text = gen_text.lstrip('\n') if destination_path: - f.write("{}\n{}".format(gen_text, sample_delim), encoding='utf-8') + f.write("{}\n{}".format(gen_text, sample_delim)) if not return_as_list and not destination_path: print("{}\n{}".format(gen_text, sample_delim), end='') gen_texts.append(gen_text) From d529426c8d9bc9ed25faca70c5c083dfb4b856f5 Mon Sep 17 00:00:00 2001 From: Larry Bowden Date: Sun, 21 Mar 2021 15:05:43 -0500 Subject: [PATCH 3/3] Added utf-8 to training encoding during file write when samples are generated --- gpt_2_simple/gpt_2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gpt_2_simple/gpt_2.py b/gpt_2_simple/gpt_2.py index 1134a26..9b48549 100644 --- a/gpt_2_simple/gpt_2.py +++ b/gpt_2_simple/gpt_2.py @@ -302,7 +302,7 @@ def generate_samples(): maketree(os.path.join(SAMPLE_DIR, run_name)) with open( os.path.join(SAMPLE_DIR, run_name, - 'samples-{}').format(counter), 'w') as fp: + 'samples-{}').format(counter), 'w', encoding='utf-8') as fp: fp.write('\n'.join(all_text)) def sample_batch():