diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..48ab6b7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,129 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don’t work, or not +# install all needed dependencies. +#Pipfile.lock + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# Vscode +.vscode/ + +# Binaries +tic.out diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..bc83780 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,22 @@ +# Works with 3.11 and tested through 3.15 +cmake_minimum_required(VERSION 3.11...3.16) + +# Project name and a few useful settings. Other commands can pick up the results +project(featex + VERSION 0.1 + DESCRIPTION "featex" + LANGUAGES C) + +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + +# Only do these if this is the main project, and not if it is included through add_subdirectory +if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) + + # Nicely support folders in IDE's + set_property(GLOBAL PROPERTY USE_FOLDERS ON) + +endif() + +add_subdirectory(src) diff --git a/README.md b/README.md index a0489be..50ec4b5 100644 --- a/README.md +++ b/README.md @@ -1,48 +1,109 @@ # featex PocketSphinx phonetic feature extraction for intelligibility prediction and remediation -To compile and run, you need to install CMU PocketSphinx, e.g., on redhat/centos/fedora: - - sudo yum install svn autoconf libtool automake bison python-devel swig - -or, on debian/ubuntu/mint: - - sudo apt-get install subversion autoconf libtool automake bison python-dev swig - -then: - - cd - mkdir src - cd src - mkdir ps - cd ps - svn checkout svn://svn.code.sf.net/p/cmusphinx/code/trunk/sphinxbase - cd sphinxbase - ./autogen.sh - make - sudo make install - cd .. - svn checkout svn://svn.code.sf.net/p/cmusphinx/code/trunk/pocketsphinx - cd pocketsphinx - ./autogen.sh - make - sudo make install - - cd test/regression - ./test-lm.sh - -This should say, "All sub-tests passed" - - cd ~/src/ps - git clone https://github.com/jsalsman/featex - cd featex - - gcc -I /usr/local/include/pocketsphinx -I /usr/local/include/sphinxbase \ - -I ~/src/ps/pocketsphinx/src/libpocketsphinx \ - -o featex featex.c -lpocketsphinx -lsphinxbase -lm - - LD_LIBRARY_PATH=/usr/local/lib ./featex we drank tea in the afternoon and watched tv - -Make sure those `-I` include paths and the `#define MODELDIR` directive near the top of featex.c correctly identify where the include files, libraries, and en-us model files were actually installed. The numeric feature stream goes to standard output, and verbose debugging output goes to stderr, so in production you might likely run it with `2>/dev/null`. If you don't want to prepend the LD_LIBRARY_PATH, which is necessary on redhat but not debian variants, see e.g. https://serverfault.com/a/372998 - -The file `Spoken-English-Intelligibility-Remediation.pdf` -- also at http://arxiv.org/abs/1709.01713 -- has more information. +### Dependencies + +#### Packages + +To compile and run, you need to install CMU PocketSphinx +On Redhat/CentOS/Fedora: + + sudo yum install svn autoconf libtool automake bison python-devel swig gcc make + +or on Debian based systems: + + sudo apt-get install subversion autoconf libtool automake bison python-dev swig gcc make + +#### Sphinxbase and Pocketsphinx + +Install both sphinxbase and pocketsphinx. Go to any temp directory and execute the following. + +Sphinxbase: + + $ svn checkout svn://svn.code.sf.net/p/cmusphinx/code/trunk/sphinxbase + $ cd sphinxbase + $ ./autogen.sh + $ make + $ sudo make install + +Pocketsphinx: + + $ svn checkout svn://svn.code.sf.net/p/cmusphinx/code/trunk/pocketsphinx + $ cd pocketsphinx + $ ./autogen.sh + $ make + $ sudo make install + +Run the builtin test: + + $ cd test/regression + $ ./test-lm.sh + +This should say, "All sub-tests passed" + +### Compiling + +Clone the repository: + + $ git clone https://github.com/jarulsamy/featex + +Create a build directory and `cd` into it: + + $ cd featex + $ mkdir build + $ cd build + +Generate the build files with `cmake`: + + $ cmake .. + +Finally, compile: + + $ make + +>The output binary, `featex.o` should be in the `bin/` directory. + + +### Usage + +View the built-in help with: + + $ ./featex --help + + Usage: featex.o [OPTION...] + featex -- PocketSphinx phonetic feature extraction for intelligibility + prediction and remediation + + -c, --combo=COMBO_PATH Path to combo.dict. + -d, --diphones Toggle play diphones + -i, --infile=INFILE_PATH Path to input raw file. + -p, --phonemes Toggle play phonemes + -P, --phrase='PHRASE' Input phrase + -t, --triphones Toggle play triphones + -u, --utterance Toggle play utterances + -w, --word Toggle play words + -?, --help Give this help list + --usage Give a short usage message + -V, --version Print program version + + Mandatory or optional arguments to long options are also mandatory or optional + for any corresponding short options. + + Report bugs to + +>The input phrase **MUST** be surrounded by quotes. + +>If you run into any missing libs while trying to run, try this: +>export LD_LIBRARY_PATH=/usr/local/lib + +The numeric feature stream goes to standard output, and verbose debugging output goes to stderr, so in production you will likely have to run it with `2>/dev/null`. If you don't want to prepend the LD_LIBRARY_PATH, which is necessary on redhat but not debian variants, see e.g. https://serverfault.com/a/372998 + +More info is documented in this [paper](assets/Spoken-English-Intelligibility-Remediation.pdf), and also [here](http://arxiv.org/abs/1709.01713). + +A demo can be run using `assets/combo.dict` and `assets/featex.raw`: + + $ ./featex.o -c assets/combo.dict -i assets/featex.raw -P 'we drank tea in the afternoon and watched tv` + +If you only want to see the end result and filter out all the debug data, redirect `stderr`: + + $ ./featex.o -c assets/combo.dict -i assets/featex.raw -P 'we drank tea in the afternoon and watched tv` 2> /dev/null diff --git a/Spoken-English-Intelligibility-Remediation.pdf b/assets/Spoken-English-Intelligibility-Remediation.pdf similarity index 100% rename from Spoken-English-Intelligibility-Remediation.pdf rename to assets/Spoken-English-Intelligibility-Remediation.pdf diff --git a/because-00766t-2.wav b/assets/because-00766t-2.wav similarity index 100% rename from because-00766t-2.wav rename to assets/because-00766t-2.wav diff --git a/because-01004t-5.wav b/assets/because-01004t-5.wav similarity index 100% rename from because-01004t-5.wav rename to assets/because-01004t-5.wav diff --git a/combo.dict b/assets/combo.dict similarity index 100% rename from combo.dict rename to assets/combo.dict diff --git a/featex-tran.txt b/assets/featex-tran.txt similarity index 100% rename from featex-tran.txt rename to assets/featex-tran.txt diff --git a/featex.raw b/assets/featex.raw similarity index 100% rename from featex.raw rename to assets/featex.raw diff --git a/example-for-fig3.py b/example-for-fig3.py deleted file mode 100644 index c77a1cd..0000000 --- a/example-for-fig3.py +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env python2 -# -*- coding: utf-8 -*- -""" -Created on Sun Aug 20 - -@author: jsalsman -""" - -from keras.models import Sequential -from keras.layers import Dense, Dropout -from keras.utils.np_utils import to_categorical -from numpy import asarray -from scipy.stats import rankdata - -lines = [] -with open('featex-tran.txt', 'r') as f: # or, phrase-sliced.txt which has some - # overlapping words so maybe rename - # one of them if you want to use the - # same server for the phrase as for - # the 82 words - lines.extend(f.readlines()) - -X = [] # testing data independents -y = [] # testing data dependents -word = '' # word name -model = {} -n = 0 -layers = 4 -units = 32 -epochs = 1000 -drop = 0.25 -features = None - -for line in lines + ['.']: - tokens = line.strip().split() - if line[0] != ' ': # new word - if word != '': # not the first word - print ("word:", word, n, "transcripts,", features, "features") - - y_cat = to_categorical(y) - model[word].fit(X, y_cat, epochs=epochs, verbose=0) - - # now you can get the probability of intelligibility for some - # featex vector Z this way: - # pi = model[word].predict(asarray(Z).reshape(1, -1))[0][1] - - if line != '.': # not the last line - word = tokens[2] - features = int(tokens[4]) * int(tokens[6]) + int(tokens[8]) - X = []; y = []; n = 0 - - model[word] = Sequential() # DNN - model[word].add(Dense(units, input_dim=features, - activation='softmax', - kernel_initializer='glorot_uniform')) - model[word].add(Dropout(drop)) - for i in range(layers): - model[word].add(Dense(units, - kernel_initializer='glorot_uniform')) - model[word].add(Dropout(drop)) - model[word].add(Dense(2, activation='softmax', - kernel_initializer='glorot_uniform')) - model[word].compile(optimizer='adam', - loss='categorical_crossentropy') - - else: # read a transcription's word data observation - if len(tokens) > features + 1: # ignore incomplete recognition results - fvec = [] - for i in range(features): - fvec.append(float(tokens[i + 2])) - if tokens[1] == "<-": - X.append(fvec) - y.append(float(tokens[0])) - n += 1 - - -because_00766t_2 = [0.22, 0.178, 0.929, 0.744, 0.05, 0.200, 0.381, 0.981, - 0.05, 0.182, 0.548, 0.000, 0.25, 0.161, 0.786, 0.512, - 0.43, 0.150, 0.929, 0.869, 0.725] -# unintelligible, pronounced "cuz" without the "bee-" -# 0.049262498 - -because_01004t_5 = [0.08, 0.277, 1.000, 0.000, 0.09, 0.275, 0.976, 0.000, - 0.11, 0.261, 0.952, 0.988, 0.06, 0.198, 0.929, 1.000, - 0.05, 0.181, 0.333, 0.919, 0.569] -# intelligible, pronounced "because-ah" as in a typical Chinese primary ESL student accent -# 0.57494467 - -def perturb(V, word): - print(model[word].predict(asarray(V).reshape(1, -1))[0][1]) - phonemes = (len(V) - 1) // 4 - pbs = [] - for n in range(phonemes): - Z = list(V) - Z[n*4 + 1] *= 1.5 - Z[n*4 + 2] *= 1.5 - Z[n*4 + 3] *= 1.5 - p_i = model[word].predict(asarray(Z).reshape(1, -1))[0][1] - print(p_i) - pbs.append(p_i) - return [int(i) for i in rankdata(pbs)] - -model['because'].predict(asarray([because_00766t_2]).reshape(1, -1))[0][1] -# 0.049262498 - -model['because'].predict(asarray([because_01004t_5]).reshape(1, -1))[0][1] -# 0.57494467 - -perturb(because_00766t_2, 'because') diff --git a/featex.c b/featex.c deleted file mode 100644 index 3fd7f06..0000000 --- a/featex.c +++ /dev/null @@ -1,457 +0,0 @@ - -// featex.c - PocketSphinx phonetic feature extraction for intelligibility prediction and remediation -// by James Salsman, July-August 2017 -// released under the MIT open source license - -#define INFILENAME "featex.raw" -#define FRATE 65 -#define MODELDIR "/usr/local/share/pocketsphinx/model/en-us/en-us" -#define DICTNAME "combo.dict" - -#include -#include "ps_alignment.h" -#include "state_align_search.h" -#include "pocketsphinx_internal.h" -#include "ps_search.h" - -#include -#include - -int -main(int argc, char *argv[]) -{ - ps_decoder_t *ps; - dict_t *dict; - dict2pid_t *d2p; - acmod_t *acmod; - bin_mdef_t *mdef; - ps_alignment_t *al; - ps_alignment_iter_t *itor, *itor2; - ps_search_t *search; - cmd_ln_t *config; - FILE *rawfh; - int16 buf[2048]; - char *fbuf, *fbip, *obuf; - size_t nread; - int16 const *bptr; - int sz, nfr, wend, n, maxdur, i, j, k, play, found; - ps_alignment_entry_t *ae; - char grammar[1000], target[10], frates[10]; - char *p, *q, *r; // string manipulation pointers for constructing grammar - ps_nbest_t *nb; - int32 score; - hash_table_t *hyptbl; - double frated; - - struct { - int start, dur, cipid, score; - } *algn; - - if (argc < 2 || (*argv[1] == '-' && (*(argv[1]+1) != 'p' || argc < 3))) { - fprintf(stderr, "usage: %s [-p[u][w][p][t][d]] word....\n" - "-p: play [u]tterance, [w]ord(s), [p]honemes (default), " - "[t]riphones, and/or [d]iphones.\n", argv[0]); - return 1; - } - - play = 0; // by default play nothing - if (*argv[1] == '-' && *(argv[1]+1) == 'p') { - play = 4; // just '-p' means to only play phonemes - if (*(argv[1]+2)) { - play = 0; - p = argv[1]+1; - while (*++p) { - if (*p == 'u') play |= 1; // utterance - else if (*p == 'w') play |= 2; // word(s) - else if (*p == 'p') play |= 4; // phonemes - else if (*p == 't') play |= 8; // triphones - else if (*p == 'd') play |= 16; // diphones - else { - fprintf(stderr, "%s: unrecogized -p option selection;\n" - "-p: play [u]tterance, [w]ord(s), [p]honemes (default)," - " [t]riphones, and/or [d]iphones.\n", argv[0]); - return 1; - } - } - } - i = 2; - } else { - i = 1; - } - -#define FPS (16000 / FRATE * 2) - - sprintf(frates, "%d", FRATE); - frated = (double) FRATE; - config = cmd_ln_init(NULL, ps_args(), FALSE, - "-hmm", MODELDIR, - "-dict", DICTNAME, - "-samprate", "16000", - "-topn", "64", // TODO parameterize for proper optimization - "-beam", "1e-57", - "-wbeam", "1e-56", - "-maxhmmpf", "-1", - "-frate", frates, - "-fsgusefiller", "no", - NULL); - if (!(ps = ps_init(config))) { - fprintf(stderr, "%s: ps_init() failed.\n", argv[0]); - return 2; - } - dict = ps->dict; - d2p = ps->d2p; - acmod = ps->acmod; - mdef = acmod->mdef; - - al = ps_alignment_init(d2p); - ps_alignment_add_word(al, dict_wordid(dict, ""), 0); - while (i < argc) { - n = dict_wordid(dict, argv[i]); - if (n < 0) { - fprintf(stderr, "%s: unrecogized word: %s\n", argv[0], argv[i]); - return 3; - } - ps_alignment_add_word(al, n, 0); - i++; - } - ps_alignment_add_word(al, dict_wordid(dict, ""), 0); - ps_alignment_populate(al); - - search = state_align_search_init("state_align", config, acmod, al); - - rawfh = fopen(INFILENAME, "rb"); - if (!rawfh) { - fprintf(stderr, "%s: can't open audio input file: %s\n", - argv[0], INFILENAME); - return 4; - } - fseek(rawfh, 0L, SEEK_END); - sz = ftell(rawfh); - fbuf = fbip = malloc(sz); - rewind(rawfh); - while (!feof(rawfh)) { - nread = fread(buf, sizeof(*buf), 2048, rawfh); - memcpy(fbip, buf, nread * sizeof(*buf)); - fbip += nread * sizeof(*buf); - } - - acmod_start_utt(acmod); - ps_search_start(search); - - bptr = (const int16 *) fbuf; - nread = (fbip - fbuf) / sizeof(*buf); - while ((nfr = acmod_process_raw(acmod, &bptr, &nread, TRUE)) > 0) { - while (acmod->n_feat_frame > 0) { - ps_search_step(search, acmod->output_frame); - acmod_advance(acmod); - } - fprintf(stderr, "%s: processed %d frames\n", argv[0], nfr); - } - - acmod_end_utt(acmod); - ps_search_finish(search); - - fprintf(stderr, "%s: aligned %d words, %d phones, and %d states\n", - argv[0], ps_alignment_n_words(al), ps_alignment_n_phones(al), - ps_alignment_n_states(al)); - - if (play & 1) { // play utterance - rawfh = fopen("/tmp/outphone.raw", "wb"); - fwrite(fbuf, sz, 1, rawfh); - fclose(rawfh); - system("play -q -r16k -ts16 -c1 /tmp/outphone.raw"); - remove("/tmp/outphone.raw"); - } - - algn = malloc(sizeof(*algn) * ps_alignment_n_phones(al)); - obuf = malloc(8000); - memset(obuf, 0, 8000); - n = 0; - - maxdur = 0; - - itor = ps_alignment_words(al); - while (itor) { - ae = ps_alignment_iter_get(itor); - fprintf(stderr, "%s: word '%s': %.2fs for %.2fs, score %d\n", argv[0], - dict->word[ae->id.wid].word, ae->start / frated, - ae->duration / frated, ae->score); - if (play & 2) { // play words - rawfh = fopen("/tmp/outphone.raw", "wb"); - fwrite(obuf, 8000, 1, rawfh); - fwrite(fbuf + ae->start * FPS, ae->duration * FPS, 1, rawfh); - fwrite(obuf, 8000, 1, rawfh); - fclose(rawfh); - system("play -q -r16k -ts16 -c1 /tmp/outphone.raw"); - remove("/tmp/outphone.raw"); - } - itor2 = ps_alignment_iter_down(itor); - wend = ae->duration + ae->start; - while (itor2) { - ae = ps_alignment_iter_get(itor2); - if (ae->start >= wend) break; - fprintf(stderr, "%s: sub-phone '%s': %.2fs for %.2fs, score %d\n", - argv[0], mdef->ciname[ae->id.pid.cipid], ae->start / frated, - ae->duration / frated, ae->score); - algn[n].start = ae->start; - algn[n].dur = ae->duration; - algn[n].score = ae->score; - algn[n++].cipid = ae->id.pid.cipid; - if (ae->duration > maxdur) - maxdur = ae->duration; - itor2 = ps_alignment_iter_next(itor2); - } - itor = ps_alignment_iter_next(itor); - } - - ps_search_free(search); - ps_alignment_free(al); - free(obuf); - - obuf = malloc(16000 + FPS * maxdur * 3); - memset(obuf, 0, 8000); - for (i = 0; i < n; i++) { - memcpy(obuf + 8000, fbuf + algn[i].start * FPS, algn[i].dur * FPS); - memset(obuf + 8000 + algn[i].dur * FPS, 0, 8000); - - fprintf(stderr, "%s: phoneme %d: %s %.2fs for %.2fs, score %d\n", - argv[0], i + 1, mdef->ciname[algn[i].cipid], - algn[i].start / frated, algn[i].dur / frated, algn[i].score); - if (play & 4) { // play phonemes (default if '-p' specified) - rawfh = fopen("/tmp/outphone.raw", "wb"); - fwrite(obuf, 16000 + algn[i].dur * FPS, 1, rawfh); - fclose(rawfh); - system("play -q -r16k -ts16 -c1 /tmp/outphone.raw"); - remove("/tmp/outphone.raw"); - } - } - - hyptbl = hash_table_new(175, HASH_CASE_YES); // for hypothesis deduplication - - for (i = 1; i < n; i++) { - - if (i == n-1) goto lastdiphone; - - if (i > 1) printf(" "); - - printf("%.2f %.3f", algn[i].dur / frated, 1 / log(2 - algn[i].score)); - - memcpy(obuf + 8000, fbuf + algn[i-1].start * FPS, - (algn[i-1].dur + algn[i].dur + algn[i+1].dur) * FPS); - memset(obuf + 8000 - + (algn[i-1].dur + algn[i].dur + algn[i+1].dur) * FPS, - 0, 8000); - - fprintf(stderr, "%s: triphone %d: %s-%s-%s\n", argv[0], i, - mdef->ciname[algn[i-1].cipid], - mdef->ciname[algn[i].cipid], - mdef->ciname[algn[i+1].cipid]); - if (play & 8) { // play triphones - rawfh = fopen("/tmp/outphone.raw", "wb"); - fwrite(obuf, 16000 + (algn[i-1].dur + algn[i].dur + - algn[i+1].dur) * FPS, 1, rawfh); - fclose(rawfh); - system("play -q -r16k -ts16 -c1 /tmp/outphone.raw"); - remove("/tmp/outphone.raw"); - } - - grammar[0] = '\0'; - strcat(grammar, "#JSGF V1.0;\ngrammar subalts;\npublic = sil1 "); - if (algn[i-1].cipid != mdef->sil) { - p = mdef->ciname[algn[i-1].cipid]; - q = grammar; - while (*++q); - while (*p) *q++ = tolower(*p++); - *q++ = '2'; - *q = '\0'; - } - strcat(grammar, " [ aa3 | ae3 | ah3 | ao3 | aw3 | ay3 | b3 | ch3 | d3" - " | dh3 | eh3 | er3 | ey3 | f3 | g3 | hh3 | ih3 | iy3 | jh3" - " | k3 | l3 | m3 | n3 | ng3 | ow3 | oy3 | p3 | r3 | s3 | sh3" - " | sil3 | t3 | th3 | uh3 | uw3 | v3 | w3 | y3 | z3 | zh3 ] "); - if (algn[i+1].cipid != mdef->sil) { - p = mdef->ciname[algn[i+1].cipid]; - q = grammar; - while (*++q); - while (*p) *q++ = tolower(*p++); - *q++ = '4'; - *q = '\0'; - } - strcat(grammar, " sil5 ;\n"); - - fprintf(stderr, "%s: %s", argv[0], grammar); - - ps_set_jsgf_string(ps, "subalts", grammar); - ps_set_search(ps, "subalts"); - ps_start_utt(ps); - ps_process_raw(ps, (const int16 *) obuf, 8000 + // samples not bytes - (algn[i-1].dur + algn[i].dur + algn[i+1].dur) * 160, - FALSE, TRUE); - ps_end_utt(ps); - - nb = ps_nbest(ps); - j = found = 0; - target[0] = ' '; target[1] = '\0'; - strcat(target, mdef->ciname[algn[i].cipid]); - strcat(target, "3"); - p = target; - while (*++p) { *p = tolower(*p); } - while (nb) { - p = (char *) ps_nbest_hyp(nb, &score); - if (p) { // some hypotheses are literally NULL - q = p; - while (*++q); - if (*(q-1) == '5') { // ignore hypotheses w/o whole match - - // ignore repeated hypotheses - if (hash_table_lookup(hyptbl, p, NULL) == -1) { - j++; - fprintf(stderr, "%s: triphone hypothesis %d: %s, %d\n", - argv[0], j, p, score); - hash_table_enter_int32(hyptbl, p, score); - - if (strstr(p, target)) { - found++; - ps_nbest_free(nb); - break; - } - } - } - } - nb = ps_nbest_next(nb); - } - if (!found) j = 42; // zero for bad recognition results or no match - fprintf(stderr, "%s: SUBSTITUTION: %.3f\n", argv[0], (42.0 - j) / 42.0); - printf(" %.3f", (42.0 - j) / 42.0); - - hash_table_empty(hyptbl); - - lastdiphone: // goto target for the final set of two phonemes - - memcpy(obuf + 8000, fbuf + algn[i-1].start * FPS, - (algn[i-1].dur + algn[i].dur) * FPS); - memset(obuf + 8000 + (algn[i-1].dur + algn[i].dur) * FPS, 0, 8000); - - fprintf(stderr, "%s: diphone %d: %s-%s\n", argv[0], i, - mdef->ciname[algn[i-1].cipid], - mdef->ciname[algn[i].cipid]); - if (play & 16) { // play diphones - rawfh = fopen("/tmp/outphone.raw", "wb"); - fwrite(obuf, 16000 + (algn[i-1].dur + algn[i].dur) * FPS, 1, rawfh); - fclose(rawfh); - system("play -q -r16k -ts16 -c1 /tmp/outphone.raw"); - remove("/tmp/outphone.raw"); - } - - grammar[0] = '\0'; - strcat(grammar, - "#JSGF V1.0;\ngrammar insdels;\npublic = sil1 [ "); - p = mdef->ciname[algn[i-1].cipid]; - q = grammar; - while (*++q); - while (*p) *q++ = tolower(*p++); - *q++ = '2'; - *q = '\0'; - r = q; - strcat(grammar, " ] [ aa3| ae3 | ah3 | ao3 | aw3 | ay3 | b3 | ch3" - " | d3 | dh3 | eh3 | er3 | ey3 | f3 | g3 | hh3 | ih3 | iy3" - " | jh3 | k3 | l3 | m3 | n3 | ng3 | ow3 | oy3 | p3 | r3 " - " | s3 | sh3 | sil3 | t3 | th3 | uh3 | uw3 | v3 | w3 | y3 " - " | z3 | zh3 ] "); - p = mdef->ciname[algn[i-1].cipid]; // first in diphone - while (*++q) { // blank out expected phoneme from possible insertions - if (isalpha(*q)) { - if ((*q == tolower(*p)) - && (((*(q+1) == '3') && *(p+1) == '\0') - || *(q+1) == tolower(*(p+1)))) { - *(q-2) = ' '; // blank out preceding '|' - *q = ' '; *(q+1) = ' '; *(q+2) = ' '; *(q+3) = ' '; - } else { - q += 3; // advance past the rest of the phoneme - } - } - } - p = mdef->ciname[algn[i].cipid]; // second in diphone - q = r; - while (*++q) { // blank out expected phoneme from possible insertions - if (isalpha(*q)) { - if ((*q == tolower(*p)) - && (((*(q+1) == '3') && *(p+1) == '\0') - || *(q+1) == tolower(*(p+1)))) { - *(q-2) = ' '; // blank out preceding '|' - *q = ' '; *(q+1) = ' '; *(q+2) = ' '; *(q+3) = ' '; - } else { - q += 3; // advance past the rest of the phoneme - } - } - } - if (algn[i].cipid != mdef->sil) { - p = mdef->ciname[algn[i].cipid]; - q = grammar; - while (*++q); - while (*p) *q++ = tolower(*p++); - *q++ = '4'; - *q = '\0'; - } - strcat(grammar, " sil5 ;\n"); - - fprintf(stderr, "%s: %s", argv[0], grammar); - - ps_set_jsgf_string(ps, "insdels", grammar); - ps_set_search(ps, "insdels"); - ps_start_utt(ps); - ps_process_raw(ps, (const int16 *) obuf, 8000 + // samples not bytes - (algn[i-1].dur + algn[i].dur) * 160, FALSE, TRUE); - ps_end_utt(ps); - - nb = ps_nbest(ps); - j = k = found = 0; - while (nb) { - p = (char *) ps_nbest_hyp(nb, &score); - if (p) { // some hypotheses are literally NULL - q = p; - while (*++q); - if (*(q-1) == '5') { // ignore hypotheses w/o whole match - - // ignore repeated hypotheses - if (hash_table_lookup(hyptbl, p, NULL) == -1) { - j++; - fprintf(stderr, "%s: diphone hypothesis %d: %s, %d\n", - argv[0], j, p, score); - hash_table_enter_int32(hyptbl, p, score); - - if (!strstr(p, "2 ")) k++; - if (strstr(p, "3 ")) k++; - - if (strstr(p, "2 ") && !strstr(p, "3 ")) { - found++; - ps_nbest_free(nb); - break; - } - } - } - } - nb = ps_nbest_next(nb); - } - if (j == 0) - k = 160; // zero for bad recognition results - else if (!found) { - k += 80; // add half the range if the preferred hypothesis missed - if (k > 160) k = 160; // clamp - } - fprintf(stderr, "%s: INS/DEL: %.3f\n", argv[0], (160.0 - k) / 160); - printf(" %.3f", (160.0 - k) / 160.0); - - hash_table_empty(hyptbl); - } - - printf("\n"); - - free(obuf); - free(algn); - free(fbuf); - ps_free(ps); - cmd_ln_free_r(config); - - return 0; -} diff --git a/include/acmod.h b/include/acmod.h new file mode 100644 index 0000000..f4d5761 --- /dev/null +++ b/include/acmod.h @@ -0,0 +1,466 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 2008 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +/** + * @file acmod.h Acoustic model structures for PocketSphinx. + * @author David Huggins-Daines + */ + +#ifndef __ACMOD_H__ +#define __ACMOD_H__ + +/* System headers. */ +#include + +/* SphinxBase headers. */ +#include +#include +#include +#include +#include +#include +#include + +/* Local headers. */ +#include "ps_mllr.h" +#include "bin_mdef.h" +#include "tmat.h" +#include "hmm.h" + +/** + * States in utterance processing. + */ +typedef enum acmod_state_e { + ACMOD_IDLE, /**< Not in an utterance. */ + ACMOD_STARTED, /**< Utterance started, no data yet. */ + ACMOD_PROCESSING, /**< Utterance in progress. */ + ACMOD_ENDED /**< Utterance ended, still buffering. */ +} acmod_state_t; + +/** + * Dummy senone score value for unintentionally active states. + */ +#define SENSCR_DUMMY 0x7fff + +/** + * Feature space linear transform structure. + */ +struct ps_mllr_s { + int refcnt; /**< Reference count. */ + int n_class; /**< Number of MLLR classes. */ + int n_feat; /**< Number of feature streams. */ + int *veclen; /**< Length of input vectors for each stream. */ + float32 ****A; /**< Rotation part of mean transformations. */ + float32 ***b; /**< Bias part of mean transformations. */ + float32 ***h; /**< Diagonal transformation of variances. */ + int32 *cb2mllr; /**< Mapping from codebooks to transformations. */ +}; + +/** + * Acoustic model parameter structure. + */ +typedef struct ps_mgau_s ps_mgau_t; + +typedef struct ps_mgaufuncs_s { + char const *name; + + int (*frame_eval)(ps_mgau_t *mgau, + int16 *senscr, + uint8 *senone_active, + int32 n_senone_active, + mfcc_t ** feat, + int32 frame, + int32 compallsen); + int (*transform)(ps_mgau_t *mgau, + ps_mllr_t *mllr); + void (*free)(ps_mgau_t *mgau); +} ps_mgaufuncs_t; + +struct ps_mgau_s { + ps_mgaufuncs_t *vt; /**< vtable of mgau functions. */ + int frame_idx; /**< frame counter. */ +}; + +#define ps_mgau_base(mg) ((ps_mgau_t *)(mg)) +#define ps_mgau_frame_eval(mg,senscr,senone_active,n_senone_active,feat,frame,compallsen) \ + (*ps_mgau_base(mg)->vt->frame_eval) \ + (mg, senscr, senone_active, n_senone_active, feat, frame, compallsen) +#define ps_mgau_transform(mg, mllr) \ + (*ps_mgau_base(mg)->vt->transform)(mg, mllr) +#define ps_mgau_free(mg) \ + (*ps_mgau_base(mg)->vt->free)(mg) + +/** + * Acoustic model structure. + * + * This object encapsulates all stages of acoustic processing, from + * raw audio input to acoustic score output. The reason for grouping + * all of these modules together is that they all have to "agree" in + * their parameterizations, and the configuration of the acoustic and + * dynamic feature computation is completely dependent on the + * parameters used to build the original acoustic model (which should + * by now always be specified in a feat.params file). + * + * Because there is not a one-to-one correspondence from blocks of + * input audio or frames of input features to frames of acoustic + * scores (due to dynamic feature calculation), results may not be + * immediately available after input, and the output results will not + * correspond to the last piece of data input. + * + * TODO: In addition, this structure serves the purpose of queueing + * frames of features (and potentially also scores in the future) for + * asynchronous passes of recognition operating in parallel. + */ +struct acmod_s { + /* Global objects, not retained. */ + cmd_ln_t *config; /**< Configuration. */ + logmath_t *lmath; /**< Log-math computation. */ + glist_t strings; /**< Temporary acoustic model filenames. */ + + /* Feature computation: */ + fe_t *fe; /**< Acoustic feature computation. */ + feat_t *fcb; /**< Dynamic feature computation. */ + + /* Model parameters: */ + bin_mdef_t *mdef; /**< Model definition. */ + tmat_t *tmat; /**< Transition matrices. */ + ps_mgau_t *mgau; /**< Model parameters. */ + ps_mllr_t *mllr; /**< Speaker transformation. */ + + /* Senone scoring: */ + int16 *senone_scores; /**< GMM scores for current frame. */ + bitvec_t *senone_active_vec; /**< Active GMMs in current frame. */ + uint8 *senone_active; /**< Array of deltas to active GMMs. */ + int senscr_frame; /**< Frame index for senone_scores. */ + int n_senone_active; /**< Number of active GMMs. */ + int log_zero; /**< Zero log-probability value. */ + + /* Utterance processing: */ + mfcc_t **mfc_buf; /**< Temporary buffer of acoustic features. */ + mfcc_t ***feat_buf; /**< Temporary buffer of dynamic features. */ + FILE *rawfh; /**< File for writing raw audio data. */ + FILE *mfcfh; /**< File for writing acoustic feature data. */ + FILE *senfh; /**< File for writing senone score data. */ + FILE *insenfh; /**< Input senone score file. */ + long *framepos; /**< File positions of recent frames in senone file. */ + + /* Rawdata collected during decoding */ + int16 *rawdata; + int32 rawdata_size; + int32 rawdata_pos; + + /* A whole bunch of flags and counters: */ + uint8 state; /**< State of utterance processing. */ + uint8 compallsen; /**< Compute all senones? */ + uint8 grow_feat; /**< Whether to grow feat_buf. */ + uint8 insen_swap; /**< Whether to swap input senone score. */ + + frame_idx_t utt_start_frame; /**< Index of the utterance start in the stream, all timings are relative to that. */ + + frame_idx_t output_frame; /**< Index of next frame of dynamic features. */ + frame_idx_t n_mfc_alloc; /**< Number of frames allocated in mfc_buf */ + frame_idx_t n_mfc_frame; /**< Number of frames active in mfc_buf */ + frame_idx_t mfc_outidx; /**< Start of active frames in mfc_buf */ + frame_idx_t n_feat_alloc; /**< Number of frames allocated in feat_buf */ + frame_idx_t n_feat_frame; /**< Number of frames active in feat_buf */ + frame_idx_t feat_outidx; /**< Start of active frames in feat_buf */ +}; +typedef struct acmod_s acmod_t; + +/** + * Initialize an acoustic model. + * + * @param config a command-line object containing parameters. This + * pointer is not retained by this object. + * @param lmath global log-math parameters. + * @param fe a previously-initialized acoustic feature module to use, + * or NULL to create one automatically. If this is supplied + * and its parameters do not match those in the acoustic + * model, this function will fail. This pointer is not retained. + * @param fe a previously-initialized dynamic feature module to use, + * or NULL to create one automatically. If this is supplied + * and its parameters do not match those in the acoustic + * model, this function will fail. This pointer is not retained. + * @return a newly initialized acmod_t, or NULL on failure. + */ +acmod_t *acmod_init(cmd_ln_t *config, logmath_t *lmath, fe_t *fe, feat_t *fcb); + +/** + * Adapt acoustic model using a linear transform. + * + * @param mllr The new transform to use, or NULL to update the existing + * transform. The decoder retains ownership of this pointer, + * so you should not attempt to free it manually. Use + * ps_mllr_retain() if you wish to reuse it + * elsewhere. + * @return The updated transform object for this decoder, or + * NULL on failure. + */ +ps_mllr_t *acmod_update_mllr(acmod_t *acmod, ps_mllr_t *mllr); + +/** + * Start logging senone scores to a filehandle. + * + * @param acmod Acoustic model object. + * @param logfh Filehandle to log to. + * @return 0 for success, <0 on error. + */ +int acmod_set_senfh(acmod_t *acmod, FILE *senfh); + +/** + * Start logging MFCCs to a filehandle. + * + * @param acmod Acoustic model object. + * @param logfh Filehandle to log to. + * @return 0 for success, <0 on error. + */ +int acmod_set_mfcfh(acmod_t *acmod, FILE *logfh); + +/** + * Start logging raw audio to a filehandle. + * + * @param acmod Acoustic model object. + * @param logfh Filehandle to log to. + * @return 0 for success, <0 on error. + */ +int acmod_set_rawfh(acmod_t *acmod, FILE *logfh); + +/** + * Finalize an acoustic model. + */ +void acmod_free(acmod_t *acmod); + +/** + * Mark the start of an utterance. + */ +int acmod_start_utt(acmod_t *acmod); + +/** + * Mark the end of an utterance. + */ +int acmod_end_utt(acmod_t *acmod); + +/** + * Rewind the current utterance, allowing it to be rescored. + * + * After calling this function, the internal frame index is reset, and + * acmod_score() will return scores starting at the first frame of the + * current utterance. Currently, acmod_set_grow() must have been + * called to enable growing the feature buffer in order for this to + * work. In the future, senone scores may be cached instead. + * + * @return 0 for success, <0 for failure (if the utterance can't be + * rewound due to no feature or score data available) + */ +int acmod_rewind(acmod_t *acmod); + +/** + * Advance the frame index. + * + * This function moves to the next frame of input data. Subsequent + * calls to acmod_score() will return scores for that frame, until the + * next call to acmod_advance(). + * + * @return New frame index. + */ +int acmod_advance(acmod_t *acmod); + +/** + * Set memory allocation policy for utterance processing. + * + * @param grow_feat If non-zero, the internal dynamic feature buffer + * will expand as necessary to encompass any amount of data fed to the + * model. + * @return previous allocation policy. + */ +int acmod_set_grow(acmod_t *acmod, int grow_feat); + +/** + * TODO: Set queue length for utterance processing. + * + * This function allows multiple concurrent passes of search to + * operate on different parts of the utterance. + */ + +/** + * Feed raw audio data to the acoustic model for scoring. + * + * @param inout_raw In: Pointer to buffer of raw samples + * Out: Pointer to next sample to be read + * @param inout_n_samps In: Number of samples available + * Out: Number of samples remaining + * @param full_utt If non-zero, this block represents a full + * utterance and should be processed as such. + * @return Number of frames of data processed. + */ +int acmod_process_raw(acmod_t *acmod, + int16 const **inout_raw, + size_t *inout_n_samps, + int full_utt); + +/** + * Feed acoustic feature data into the acoustic model for scoring. + * + * @param inout_cep In: Pointer to buffer of features + * Out: Pointer to next frame to be read + * @param inout_n_frames In: Number of frames available + * Out: Number of frames remaining + * @param full_utt If non-zero, this block represents a full + * utterance and should be processed as such. + * @return Number of frames of data processed. + */ +int acmod_process_cep(acmod_t *acmod, + mfcc_t ***inout_cep, + int *inout_n_frames, + int full_utt); + +/** + * Feed dynamic feature data into the acoustic model for scoring. + * + * Unlike acmod_process_raw() and acmod_process_cep(), this function + * accepts a single frame at a time. This is because there is no need + * to do buffering when using dynamic features as input. However, if + * the dynamic feature buffer is full, this function will fail, so you + * should either always check the return value, or always pair a call + * to it with a call to acmod_score(). + * + * @param feat Pointer to one frame of dynamic features. + * @return Number of frames processed (either 0 or 1). + */ +int acmod_process_feat(acmod_t *acmod, + mfcc_t **feat); + +/** + * Set up a senone score dump file for input. + * + * @param insenfh File handle of dump file + * @return 0 for success, <0 for failure + */ +int acmod_set_insenfh(acmod_t *acmod, FILE *insenfh); + +/** + * Read one frame of scores from senone score dump file. + * + * @return Number of frames read or <0 on error. + */ +int acmod_read_scores(acmod_t *acmod); + +/** + * Get a frame of dynamic feature data. + * + * @param inout_frame_idx Input: frame index to get, or NULL + * to obtain features for the most recent frame. + * Output: frame index corresponding to this + * set of features. + * @return Feature array, or NULL if requested frame is not available. + */ +mfcc_t **acmod_get_frame(acmod_t *acmod, int *inout_frame_idx); + +/** + * Score one frame of data. + * + * @param inout_frame_idx Input: frame index to score, or NULL + * to obtain scores for the most recent frame. + * Output: frame index corresponding to this + * set of scores. + * @return Array of senone scores for this frame, or NULL if no frame + * is available for scoring (such as if a frame index is + * requested that is not yet or no longer available). The + * data pointed to persists only until the next call to + * acmod_score() or acmod_advance(). + */ +int16 const *acmod_score(acmod_t *acmod, + int *inout_frame_idx); + +/** + * Write senone dump file header. + */ +int acmod_write_senfh_header(acmod_t *acmod, FILE *logfh); + +/** + * Write a frame of senone scores to a dump file. + */ +int acmod_write_scores(acmod_t *acmod, int n_active, uint8 const *active, + int16 const *senscr, FILE *senfh); + + +/** + * Get best score and senone index for current frame. + */ +int acmod_best_score(acmod_t *acmod, int *out_best_senid); + +/** + * Clear set of active senones. + */ +void acmod_clear_active(acmod_t *acmod); + +/** + * Activate senones associated with an HMM. + */ +void acmod_activate_hmm(acmod_t *acmod, hmm_t *hmm); + +/** + * Activate a single senone. + */ +#define acmod_activate_sen(acmod, sen) bitvec_set((acmod)->senone_active_vec, sen) + +/** + * Build active list from + */ +int32 acmod_flags2list(acmod_t *acmod); + +/** + * Get the offset of the utterance start of the current stream, helpful for stream-wide timing. + */ +int32 acmod_stream_offset(acmod_t *acmod); + +/** + * Reset the current stream + */ +void acmod_start_stream(acmod_t *acmod); + +/** + * Sets the limit of the raw audio data to store + */ +void acmod_set_rawdata_size(acmod_t *acmod, int32 size); + +/** + * Retrieves the raw data collected during utterance decoding + */ +void acmod_get_rawdata(acmod_t *acmod, int16 **buffer, int32 *size); + +#endif /* __ACMOD_H__ */ diff --git a/include/bin_mdef.h b/include/bin_mdef.h new file mode 100644 index 0000000..a22aa2e --- /dev/null +++ b/include/bin_mdef.h @@ -0,0 +1,236 @@ +/* -*- c-file-style: "linux" -*- */ +/* ==================================================================== + * Copyright (c) 2005 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ +/** + * @file bin_mdef.h + * + * Binary format model definition files, with support for + * heterogeneous topologies and variable-size N-phones + * + * @author David Huggins-Daines + */ +#ifndef __BIN_MDEF_H__ +#define __BIN_MDEF_H__ + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +/* SphinxBase headers. */ +#include +#include +#include + +#include "mdef.h" + +#define BIN_MDEF_FORMAT_VERSION 1 +/* Little-endian machines will write "BMDF" to disk, big-endian ones "FDMB". */ +#define BIN_MDEF_NATIVE_ENDIAN 0x46444d42 /* 'BMDF' in little-endian order */ +#define BIN_MDEF_OTHER_ENDIAN 0x424d4446 /* 'BMDF' in big-endian order */ +#ifdef __GNUC__ +#define __ATTRIBUTE_PACKED __attribute__((packed)) +#else +#define __ATTRIBUTE_PACKED +#endif + +/** + * Phone entry (on-disk, 12 bytes) + */ +typedef struct mdef_entry_s mdef_entry_t; +struct mdef_entry_s { + int32 ssid; /**< Senone sequence ID */ + int32 tmat; /**< Transition matrix ID */ + /* FIXME: is any of this actually necessary? */ + union { + /**< CI phone information - attributes (just "filler" for now) */ + struct { + uint8 filler; + uint8 reserved[3]; + } ci; + /**< CD phone information - context info. */ + struct { + uint8 wpos; + uint8 ctx[3]; /**< quintphones will require hacking */ + } cd; + } info; +} __ATTRIBUTE_PACKED; + +/** + * Invalid senone sequence ID (limited to 16 bits for PocketSphinx). + */ +#define BAD_SSID 0xffff +/** + * Invalid senone ID (limited to 16 bits for PocketSphinx). + */ +#define BAD_SENID 0xffff + +/** + * Node in CD phone tree (on-disk, 8 bytes). + */ +typedef struct cd_tree_s cd_tree_t; +struct cd_tree_s { + int16 ctx; /**< Context (word position or CI phone) */ + int16 n_down; /**< Number of children (0 for leafnode) */ + union { + int32 pid; /**< Phone ID (leafnode) */ + int32 down; /**< Next level of the tree (offset from start of cd_trees) */ + } c; +}; + +/** + * Model definition structure (in-memory). + */ +typedef struct bin_mdef_s bin_mdef_t; +struct bin_mdef_s { + int refcnt; + int32 n_ciphone; /**< Number of base (CI) phones */ + int32 n_phone; /**< Number of base (CI) phones + (CD) triphones */ + int32 n_emit_state; /**< Number of emitting states per phone (0 for heterogeneous) */ + int32 n_ci_sen; /**< Number of CI senones; these are the first */ + int32 n_sen; /**< Number of senones (CI+CD) */ + int32 n_tmat; /**< Number of transition matrices */ + int32 n_sseq; /**< Number of unique senone sequences */ + int32 n_ctx; /**< Number of phones of context */ + int32 n_cd_tree; /**< Number of nodes in cd_tree (below) */ + int32 sil; /**< CI phone ID for silence */ + + mmio_file_t *filemap;/**< File map for this file (if any) */ + char **ciname; /**< CI phone names */ + cd_tree_t *cd_tree; /**< Tree mapping CD phones to phone IDs */ + mdef_entry_t *phone; /**< All phone structures */ + uint16 **sseq; /**< Unique senone sequences (2D array built at load time) */ + uint8 *sseq_len; /**< Number of states in each sseq (NULL for homogeneous) */ + + /* These two are not stored on disk, but are generated at load time. */ + int16 *cd2cisen; /**< Parent CI-senone id for each senone */ + int16 *sen2cimap; /**< Parent CI-phone for each senone (CI or CD) */ + + /** Allocation mode for this object. */ + enum { BIN_MDEF_FROM_TEXT, BIN_MDEF_IN_MEMORY, BIN_MDEF_ON_DISK } alloc_mode; +}; + +#define bin_mdef_is_fillerphone(m,p) (((p) < (m)->n_ciphone) \ + ? (m)->phone[p].info.ci.filler \ + : (m)->phone[(m)->phone[p].info.cd.ctx[0]].info.ci.filler) +#define bin_mdef_is_ciphone(m,p) ((p) < (m)->n_ciphone) +#define bin_mdef_n_ciphone(m) ((m)->n_ciphone) +#define bin_mdef_n_phone(m) ((m)->n_phone) +#define bin_mdef_n_sseq(m) ((m)->n_sseq) +#define bin_mdef_n_emit_state(m) ((m)->n_emit_state) +#define bin_mdef_n_emit_state_phone(m,p) ((m)->n_emit_state ? (m)->n_emit_state \ + : (m)->sseq_len[(m)->phone[p].ssid]) +#define bin_mdef_n_sen(m) ((m)->n_sen) +#define bin_mdef_n_tmat(m) ((m)->n_tmat) +#define bin_mdef_pid2ssid(m,p) ((m)->phone[p].ssid) +#define bin_mdef_pid2tmatid(m,p) ((m)->phone[p].tmat) +#define bin_mdef_silphone(m) ((m)->sil) +#define bin_mdef_sen2cimap(m,s) ((m)->sen2cimap[s]) +#define bin_mdef_sseq2sen(m,ss,pos) ((m)->sseq[ss][pos]) +#define bin_mdef_pid2ci(m,p) (((p) < (m)->n_ciphone) ? (p) \ + : (m)->phone[p].info.cd.ctx[0]) + +/** + * Read a binary mdef from a file. + */ +POCKETSPHINX_EXPORT +bin_mdef_t *bin_mdef_read(cmd_ln_t *config, const char *filename); +/** + * Read a text mdef from a file (creating an in-memory binary mdef). + */ +POCKETSPHINX_EXPORT +bin_mdef_t *bin_mdef_read_text(cmd_ln_t *config, const char *filename); +/** + * Write a binary mdef to a file. + */ +POCKETSPHINX_EXPORT +int bin_mdef_write(bin_mdef_t *m, const char *filename); +/** + * Write a binary mdef to a text file. + */ +POCKETSPHINX_EXPORT +int bin_mdef_write_text(bin_mdef_t *m, const char *filename); +/** + * Retain a pointer to a bin_mdef_t. + */ +bin_mdef_t *bin_mdef_retain(bin_mdef_t *m); +/** + * Release a pointer to a binary mdef. + */ +int bin_mdef_free(bin_mdef_t *m); + +/** + * Context-independent phone lookup. + * @return phone id for ciphone. + */ +int bin_mdef_ciphone_id(bin_mdef_t *m, /**< In: Model structure being queried */ + const char *ciphone); /**< In: ciphone for which id wanted */ + +/** + * Case-insensitive context-independent phone lookup. + * @return phone id for ciphone. + */ +int bin_mdef_ciphone_id_nocase(bin_mdef_t *m, /**< In: Model structure being queried */ + const char *ciphone); /**< In: ciphone for which id wanted */ + +/* Return value: READ-ONLY ciphone string name for the given ciphone id */ +const char *bin_mdef_ciphone_str(bin_mdef_t *m, /**< In: Model structure being queried */ + int32 ci); /**< In: ciphone id for which name wanted */ + +/* Return value: phone id for the given constituents if found, else -1 */ +int bin_mdef_phone_id(bin_mdef_t *m, /**< In: Model structure being queried */ + int32 b, /**< In: base ciphone id */ + int32 l, /**< In: left context ciphone id */ + int32 r, /**< In: right context ciphone id */ + int32 pos); /**< In: Word position */ + +/* Look up a phone id, backing off to other word positions. */ +int bin_mdef_phone_id_nearest(bin_mdef_t * m, int32 b, + int32 l, int32 r, int32 pos); + +/** + * Create a phone string for the given phone (base or triphone) id in the given buf. + * + * @return 0 if successful, -1 if error. + */ +int bin_mdef_phone_str(bin_mdef_t *m, /**< In: Model structure being queried */ + int pid, /**< In: phone id being queried */ + char *buf); /**< Out: On return, buf has the string */ + +#ifdef __cplusplus +}; /* extern "C" */ +#endif /* __cplusplus */ + +#endif /* __BIN_MDEF_H__ */ diff --git a/include/dict.h b/include/dict.h new file mode 100644 index 0000000..26ffd2b --- /dev/null +++ b/include/dict.h @@ -0,0 +1,210 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 1999-2004 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +#ifndef _S3_DICT_H_ +#define _S3_DICT_H_ + +/** \file dict.h + * \brief Operations on dictionary. + */ + +/* SphinxBase headers. */ +#include + +/* Local headers. */ +#include "s3types.h" +#include "bin_mdef.h" +#include "pocketsphinx_export.h" + +#define S3DICT_INC_SZ 4096 + +#ifdef __cplusplus +extern "C" { +#endif + +/** + \struct dictword_t + \brief a structure for one dictionary word. +*/ +typedef struct { + char *word; /**< Ascii word string */ + s3cipid_t *ciphone; /**< Pronunciation */ + int32 pronlen; /**< Pronunciation length */ + s3wid_t alt; /**< Next alternative pronunciation id, NOT_S3WID if none */ + s3wid_t basewid; /**< Base pronunciation id */ +} dictword_t; + +/** + \struct dict_t + \brief a structure for a dictionary. +*/ + +typedef struct { + int refcnt; + bin_mdef_t *mdef; /**< Model definition used for phone IDs; NULL if none used */ + dictword_t *word; /**< Array of entries in dictionary */ + hash_table_t *ht; /**< Hash table for mapping word strings to word ids */ + int32 max_words; /**< #Entries allocated in dict, including empty slots */ + int32 n_word; /**< #Occupied entries in dict; ie, excluding empty slots */ + int32 filler_start; /**< First filler word id (read from filler dict) */ + int32 filler_end; /**< Last filler word id (read from filler dict) */ + s3wid_t startwid; /**< FOR INTERNAL-USE ONLY */ + s3wid_t finishwid; /**< FOR INTERNAL-USE ONLY */ + s3wid_t silwid; /**< FOR INTERNAL-USE ONLY */ + int nocase; +} dict_t; + + +/** + * Initialize a new dictionary. + * + * If config and mdef are supplied, then the dictionary will be read + * from the files specified by the -dict and -fdict options in config, + * with case sensitivity determined by the -dictcase option. + * + * Otherwise an empty case-sensitive dictionary will be created. + * + * Return ptr to dict_t if successful, NULL otherwise. + */ +dict_t *dict_init(cmd_ln_t *config, /**< Configuration (-dict, -fdict, -dictcase) or NULL */ + bin_mdef_t *mdef /**< For looking up CI phone IDs (or NULL) */ + ); + +/** + * Write dictionary to a file. + */ +int dict_write(dict_t *dict, char const *filename, char const *format); + +/** Return word id for given word string if present. Otherwise return BAD_S3WID */ +POCKETSPHINX_EXPORT +s3wid_t dict_wordid(dict_t *d, const char *word); + +/** + * Return 1 if w is a filler word, 0 if not. A filler word is one that was read in from the + * filler dictionary; however, sentence START and FINISH words are not filler words. + */ +int dict_filler_word(dict_t *d, /**< The dictionary structure */ + s3wid_t w /**< The word ID */ + ); + +/** + * Test if w is a "real" word, i.e. neither a filler word nor START/FINISH. + */ +POCKETSPHINX_EXPORT +int dict_real_word(dict_t *d, /**< The dictionary structure */ + s3wid_t w /**< The word ID */ + ); + +/** + * Add a word with the given ciphone pronunciation list to the dictionary. + * Return value: Result word id if successful, BAD_S3WID otherwise + */ +s3wid_t dict_add_word(dict_t *d, /**< The dictionary structure. */ + char const *word, /**< The word. */ + s3cipid_t const *p, /**< The pronunciation. */ + int32 np /**< Number of phones. */ + ); + +/** + * Return value: CI phone string for the given word, phone position. + */ +const char *dict_ciphone_str(dict_t *d, /**< In: Dictionary to look up */ + s3wid_t wid, /**< In: Component word being looked up */ + int32 pos /**< In: Pronunciation phone position */ + ); + +/** Packaged macro access to dictionary members */ +#define dict_size(d) ((d)->n_word) +#define dict_num_fillers(d) (dict_filler_end(d) - dict_filler_start(d)) +/** + * Number of "real words" in the dictionary. + * + * This is the number of words that are not fillers, , or . + */ +#define dict_num_real_words(d) \ + (dict_size(d) - (dict_filler_end(d) - dict_filler_start(d)) - 2) +#define dict_basewid(d,w) ((d)->word[w].basewid) +#define dict_wordstr(d,w) ((w) < 0 ? NULL : (d)->word[w].word) +#define dict_basestr(d,w) ((d)->word[dict_basewid(d,w)].word) +#define dict_nextalt(d,w) ((d)->word[w].alt) +#define dict_pronlen(d,w) ((d)->word[w].pronlen) +#define dict_pron(d,w,p) ((d)->word[w].ciphone[p]) /**< The CI phones of the word w at position p */ +#define dict_filler_start(d) ((d)->filler_start) +#define dict_filler_end(d) ((d)->filler_end) +#define dict_startwid(d) ((d)->startwid) +#define dict_finishwid(d) ((d)->finishwid) +#define dict_silwid(d) ((d)->silwid) +#define dict_is_single_phone(d,w) ((d)->word[w].pronlen == 1) +#define dict_first_phone(d,w) ((d)->word[w].ciphone[0]) +#define dict_second_phone(d,w) ((d)->word[w].ciphone[1]) +#define dict_second_last_phone(d,w) ((d)->word[w].ciphone[(d)->word[w].pronlen - 2]) +#define dict_last_phone(d,w) ((d)->word[w].ciphone[(d)->word[w].pronlen - 1]) + +/* Hard-coded special words */ +#define S3_START_WORD "" +#define S3_FINISH_WORD "" +#define S3_SILENCE_WORD "" +#define S3_UNKNOWN_WORD "" + +/** + * If the given word contains a trailing "(....)" (i.e., a Sphinx-II style alternative + * pronunciation specification), strip that trailing portion from it. Note that the given + * string is modified. + * Return value: If string was modified, the character position at which the original string + * was truncated; otherwise -1. + */ +int32 dict_word2basestr(char *word); + +/** + * Retain a pointer to an dict_t. + */ +dict_t *dict_retain(dict_t *d); + +/** + * Release a pointer to a dictionary. + */ +int dict_free(dict_t *d); + +/** Report a dictionary structure */ +void dict_report(dict_t *d /**< A dictionary structure */ + ); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/dict2pid.h b/include/dict2pid.h new file mode 100644 index 0000000..f81cf63 --- /dev/null +++ b/include/dict2pid.h @@ -0,0 +1,180 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 1999-2014 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +#ifndef _S3_DICT2PID_H_ +#define _S3_DICT2PID_H_ + +/* System headers. */ +#include + +/* SphinxBase headers. */ +#include +#include + +/* Local headers. */ +#include "s3types.h" +#include "bin_mdef.h" +#include "dict.h" + +/** \file dict2pid.h + * \brief Building triphones for a dictionary. + * + * This is one of the more complicated parts of a cross-word + * triphone model decoder. The first and last phones of each word + * get their left and right contexts, respectively, from other + * words. For single-phone words, both its contexts are from other + * words, simultaneously. As these words are not known beforehand, + * life gets complicated. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * \struct xwdssid_t + * \brief cross word triphone model structure + */ + +typedef struct { + s3ssid_t *ssid; /**< Senone Sequence ID list for all context ciphones */ + s3cipid_t *cimap; /**< Index into ssid[] above for each ci phone */ + int32 n_ssid; /**< #Unique ssid in above, compressed ssid list */ +} xwdssid_t; + +/** + \struct dict2pid_t + \brief Building composite triphone (as well as word internal triphones) with the dictionary. +*/ + +typedef struct { + int refcount; + + bin_mdef_t *mdef; /**< Model definition, used to generate + internal ssids on the fly. */ + dict_t *dict; /**< Dictionary this table refers to. */ + + /*Notice the order of the arguments */ + /* FIXME: This is crying out for compression - in Mandarin we have + * 180 context independent phones, which makes this an 11MB + * array. */ + s3ssid_t ***ldiph_lc; /**< For multi-phone words, [base][rc][lc] -> ssid; filled out for + word-initial base x rc combinations in current vocabulary */ + + + xwdssid_t **rssid; /**< Right context state sequence id table + First dimension: base phone, + Second dimension: left context. + */ + + + s3ssid_t ***lrdiph_rc; /**< For single-phone words, [base][lc][rc] -> ssid; filled out for + single-phone base x lc combinations in current vocabulary */ + + xwdssid_t **lrssid; /**< Left-Right context state sequence id table + First dimension: base phone, + Second dimension: left context. + */ +} dict2pid_t; + +/** Access macros; not designed for arbitrary use */ +#define dict2pid_rssid(d,ci,lc) (&(d)->rssid[ci][lc]) +#define dict2pid_ldiph_lc(d,b,r,l) ((d)->ldiph_lc[b][r][l]) +#define dict2pid_lrdiph_rc(d,b,l,r) ((d)->lrdiph_rc[b][l][r]) + +/** + * Build the dict2pid structure for the given model/dictionary + */ +dict2pid_t *dict2pid_build(bin_mdef_t *mdef, /**< A model definition*/ + dict_t *dict /**< An initialized dictionary */ + ); + +/** + * Retain a pointer to dict2pid + */ +dict2pid_t *dict2pid_retain(dict2pid_t *d2p); + +/** + * Free the memory dict2pid structure + */ +int dict2pid_free(dict2pid_t *d2p /**< In: the d2p */ + ); + +/** + * Return the senone sequence ID for the given word position. + */ +s3ssid_t dict2pid_internal(dict2pid_t *d2p, + int32 wid, + int pos); + +/** + * Add a word to the dict2pid structure (after adding it to dict). + */ +int dict2pid_add_word(dict2pid_t *d2p, + int32 wid); + +/** + * For debugging + */ +void dict2pid_dump(FILE *fp, /**< In: a file pointer */ + dict2pid_t *d2p /**< In: a dict2pid_t structure */ + ); + +/** Report a dict2pid data structure */ +void dict2pid_report(dict2pid_t *d2p /**< In: a dict2pid_t structure */ + ); + +/** + * Get number of rc + */ +int32 get_rc_nssid(dict2pid_t *d2p, /**< In: a dict2pid */ + s3wid_t w /**< In: a wid */ + ); + +/** + * Get RC map + */ +s3cipid_t* dict2pid_get_rcmap(dict2pid_t *d2p, /**< In: a dict2pid */ + s3wid_t w /**< In: a wid */ + ); + +#ifdef __cplusplus +} +#endif + + +#endif diff --git a/include/hmm.h b/include/hmm.h new file mode 100644 index 0000000..2bfb462 --- /dev/null +++ b/include/hmm.h @@ -0,0 +1,306 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 1999-2004 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +/** + * @file hmm.h Hidden Markov Model base structures. + */ + +#ifndef __HMM_H__ +#define __HMM_H__ + +/* System headers. */ +#include + +/* SphinxBase headers. */ +#include +#include + +/* PocketSphinx headers. */ +#include "bin_mdef.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Type for frame index values. Used in HMM indexes and + * backpointers and affects memory required.Due to limitations of FSG + * search implementation this value needs to be signed. + */ +typedef int32 frame_idx_t; + +/** + * Maximum number of frames in index, should be in sync with above. + */ +#define MAX_N_FRAMES MAX_INT32 + + +/** Shift count for senone scores. */ +#define SENSCR_SHIFT 10 + +/** + * Large "bad" score. + * + * This number must be "bad" enough so that 4 times WORST_SCORE will + * not overflow. The reason for this is that the search doesn't check + * the scores in a model before evaluating the model and it may + * require as many was 4 plies before the new 'good' score can wipe + * out the initial WORST_SCORE initialization. + */ +#define WORST_SCORE ((int)0xE0000000) + +/** + * Watch out, though! Transition matrix entries that are supposed to + * be "zero" don't actually get that small due to quantization. + */ +#define TMAT_WORST_SCORE (-255) + +/** + * Is one score better than another? + */ +#define BETTER_THAN > + +/** + * Is one score worse than another? + */ +#define WORSE_THAN < + +/** \file hmm.h + * \brief HMM data structure and operation + * + * For efficiency, this version is hardwired for two possible HMM + * topologies, but will fall back to others: + * + * 5-state left-to-right HMMs: (0 is the *emitting* entry state and E + * is a non-emitting exit state; the x's indicate allowed transitions + * between source and destination states): + * + * 
+ *               0   1   2   3   4   E (destination-states)
+ *           0   x   x   x
+ *           1       x   x   x
+ *           2           x   x   x
+ *           3               x   x   x
+ *           4                   x   x
+ *    (source-states)
+ *
+ * + * 5-state topologies that contain a subset of the above transitions should work as well. + * + * 3-state left-to-right HMMs (similar notation as the 5-state topology above): + * + * 
+ *               0   1   2   E (destination-states)
+ *           0   x   x   x
+ *           1       x   x   x
+ *           2           x   x 
+ *    (source-states)
+ *
+ * + * 3-state topologies that contain a subset of the above transitions should work as well. + */ + +/** + * @struct hmm_context_t + * @brief Shared information between a set of HMMs. + * + * We assume that the initial state is emitting and that the + * transition matrix is n_emit_state x (n_emit_state+1), where the + * extra destination dimension correponds to the non-emitting final or + * exit state. + */ +typedef struct hmm_context_s { + int32 n_emit_state; /**< Number of emitting states in this set of HMMs. */ + uint8 ** const *tp; /**< State transition scores tp[id][from][to] (logs3 values). */ + int16 const *senscore; /**< State emission scores senscore[senid] + (negated scaled logs3 values). */ + uint16 * const *sseq; /**< Senone sequence mapping. */ + int32 *st_sen_scr; /**< Temporary array of senone scores (for some topologies). */ + listelem_alloc_t *mpx_ssid_alloc; /**< Allocator for senone sequence ID arrays. */ + void *udata; /**< Whatever you feel like, gosh. */ +} hmm_context_t; + +/** + * Hard-coded limit on the number of emitting states. + */ +#define HMM_MAX_NSTATE 5 + +/** + * @struct hmm_t + * @brief An individual HMM among the HMM search space. + * + * An individual HMM among the HMM search space. An HMM with N + * emitting states consists of N+1 internal states including the + * non-emitting exit (out) state. + */ +typedef struct hmm_s { + hmm_context_t *ctx; /**< Shared context data for this HMM. */ + int32 score[HMM_MAX_NSTATE]; /**< State scores for emitting states. */ + int32 history[HMM_MAX_NSTATE]; /**< History indices for emitting states. */ + int32 out_score; /**< Score for non-emitting exit state. */ + int32 out_history; /**< History index for non-emitting exit state. */ + uint16 ssid; /**< Senone sequence ID (for non-MPX) */ + uint16 senid[HMM_MAX_NSTATE]; /**< Senone IDs (non-MPX) or sequence IDs (MPX) */ + int32 bestscore; /**< Best [emitting] state score in current frame (for pruning). */ + int16 tmatid; /**< Transition matrix ID (see hmm_context_t). */ + frame_idx_t frame; /**< Frame in which this HMM was last active; <0 if inactive */ + uint8 mpx; /**< Is this HMM multiplex? (hoisted for speed) */ + uint8 n_emit_state; /**< Number of emitting states (hoisted for speed) */ +} hmm_t; + +/** Access macros. */ +#define hmm_context(h) (h)->ctx +#define hmm_is_mpx(h) (h)->mpx + +#define hmm_in_score(h) (h)->score[0] +#define hmm_score(h,st) (h)->score[st] +#define hmm_out_score(h) (h)->out_score + +#define hmm_in_history(h) (h)->history[0] +#define hmm_history(h,st) (h)->history[st] +#define hmm_out_history(h) (h)->out_history + +#define hmm_bestscore(h) (h)->bestscore +#define hmm_frame(h) (h)->frame +#define hmm_mpx_ssid(h,st) (h)->senid[st] +#define hmm_nonmpx_ssid(h) (h)->ssid +#define hmm_ssid(h,st) (hmm_is_mpx(h) \ + ? hmm_mpx_ssid(h,st) : hmm_nonmpx_ssid(h)) +#define hmm_mpx_senid(h,st) (hmm_mpx_ssid(h,st) == BAD_SENID \ + ? BAD_SENID : (h)->ctx->sseq[hmm_mpx_ssid(h,st)][st]) +#define hmm_nonmpx_senid(h,st) ((h)->senid[st]) +#define hmm_senid(h,st) (hmm_is_mpx(h) \ + ? hmm_mpx_senid(h,st) : hmm_nonmpx_senid(h,st)) +#define hmm_senscr(h,st) (hmm_senid(h,st) == BAD_SENID \ + ? WORST_SCORE \ + : -(h)->ctx->senscore[hmm_senid(h,st)]) +#define hmm_tmatid(h) (h)->tmatid +#define hmm_tprob(h,i,j) (-(h)->ctx->tp[hmm_tmatid(h)][i][j]) +#define hmm_n_emit_state(h) ((h)->n_emit_state) +#define hmm_n_state(h) ((h)->n_emit_state + 1) + +/** + * Create an HMM context. + **/ +hmm_context_t *hmm_context_init(int32 n_emit_state, + uint8 ** const *tp, + int16 const *senscore, + uint16 * const *sseq); + +/** + * Change the senone score array for a context. + **/ +#define hmm_context_set_senscore(ctx, senscr) ((ctx)->senscore = (senscr)) + +/** + * Free an HMM context. + * + * @note The transition matrices, senone scores, and senone sequence + * mapping are all assumed to be allocated externally, and will NOT be + * freed by this function. + **/ +void hmm_context_free(hmm_context_t *ctx); + +/** + * Populate a previously-allocated HMM structure, allocating internal data. + **/ +void hmm_init(hmm_context_t *ctx, hmm_t *hmm, int mpx, int ssid, int tmatid); + +/** + * Free an HMM structure, releasing internal data (but not the HMM structure itself). + */ +void hmm_deinit(hmm_t *hmm); + +/** + * Reset the states of the HMM to the invalid condition. + + * i.e., scores to WORST_SCORE and hist to undefined. + */ +void hmm_clear(hmm_t *h); + +/** + * Reset the scores of the HMM. + */ +void hmm_clear_scores(hmm_t *h); + +/** + * Renormalize the scores in this HMM based on the given best score. + */ +void hmm_normalize(hmm_t *h, int32 bestscr); + +/** + * Enter an HMM with the given path score and history ID. + **/ +void hmm_enter(hmm_t *h, int32 score, + int32 histid, int frame); + +/** + * Viterbi evaluation of given HMM. + * + * @note If this module were being used for tracking state + * segmentations, the dummy, non-emitting exit state would have to be + * updated separately. In the Viterbi DP diagram, transitions to the + * exit state occur from the current time; they are vertical + * transitions. Hence they should be made only after the history has + * been logged for the emitting states. But we're not bothered with + * state segmentations, for now. So, we update the exit state as + * well. +*/ +int32 hmm_vit_eval(hmm_t *hmm); + + +/** + * Like hmm_vit_eval, but dump HMM state and relevant senscr to fp first, for debugging;. + */ +int32 hmm_dump_vit_eval(hmm_t *hmm, /**< In/Out: HMM being updated */ + FILE *fp /**< An output file pointer */ + ); + +/** + * For debugging, dump the whole HMM out. + */ + +void hmm_dump(hmm_t *h, /**< In/Out: HMM being updated */ + FILE *fp /**< An output file pointer */ + ); + + +#ifdef __cplusplus +} +#endif + +#endif /* __HMM_H__ */ diff --git a/include/mdef.h b/include/mdef.h new file mode 100644 index 0000000..b0a7ced --- /dev/null +++ b/include/mdef.h @@ -0,0 +1,271 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 1999-2004 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +/* + * mdef.h -- HMM model definition: base (CI) phones and triphones + * + * ********************************************** + * CMU ARPA Speech Project + * + * Copyright (c) 1999 Carnegie Mellon University. + * ALL RIGHTS RESERVED. + * ********************************************** + */ + + +#ifndef __MDEF_H__ +#define __MDEF_H__ + + +/* System headers. */ +#include + +/* SphinxBase headers. */ +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** \file mdef.h + * \brief Model definition + */ + +/** \enum word_posn_t + * \brief Union of different type of word position + */ + +typedef enum { + WORD_POSN_INTERNAL = 0, /**< Internal phone of word */ + WORD_POSN_BEGIN = 1, /**< Beginning phone of word */ + WORD_POSN_END = 2, /**< Ending phone of word */ + WORD_POSN_SINGLE = 3, /**< Single phone word (i.e. begin & end) */ + WORD_POSN_UNDEFINED = 4 /**< Undefined value, used for initial conditions, etc */ +} word_posn_t; +#define N_WORD_POSN 4 /**< total # of word positions (excluding undefined) */ +#define WPOS_NAME "ibesu" /**< Printable code for each word position above */ +#define S3_SILENCE_CIPHONE "SIL" /**< Hard-coded silence CI phone name */ + +/** + \struct ciphone_t + \brief CI phone information +*/ +typedef struct { + char *name; /**< The name of the CI phone */ + int32 filler; /**< Whether a filler phone; if so, can be substituted by + silence phone in left or right context position */ +} ciphone_t; + +/** + * \struct phone_t + * \brief Triphone information, including base phones as a subset. For the latter, lc, rc and wpos are non-existent. + */ +typedef struct { + int32 ssid; /**< State sequence (or senone sequence) ID, considering the + n_emit_state senone-ids are a unit. The senone sequences + themselves are in a separate table */ + int32 tmat; /**< Transition matrix id */ + int16 ci, lc, rc; /**< Base, left, right context ciphones */ + word_posn_t wpos; /**< Word position */ + +} phone_t; + +/** + * \struct ph_rc_t + * \brief Structures needed for mapping into pid. (See mdef_t.wpos_ci_lclist below.) (lc = left context; rc = right context.) + * NOTE: Both ph_rc_t and ph_lc_t FOR INTERNAL USE ONLY. + */ +typedef struct ph_rc_s { + int16 rc; /**< Specific rc for a parent */ + int32 pid; /**< Triphone id for above rc instance */ + struct ph_rc_s *next; /**< Next rc entry for same parent */ +} ph_rc_t; + +/** + * \struct ph_lc_t + * \brief Structures for storing the left context. + */ + +typedef struct ph_lc_s { + int16 lc; /**< Specific lc for a parent */ + ph_rc_t *rclist; /**< rc list for above lc instance */ + struct ph_lc_s *next; /**< Next lc entry for same parent */ +} ph_lc_t; + + +/** The main model definition structure */ +/** + \struct mdef_t + \brief strcture for storing the model definition. +*/ +typedef struct { + int32 n_ciphone; /**< number basephones actually present */ + int32 n_phone; /**< number basephones + number triphones actually present */ + int32 n_emit_state; /**< number emitting states per phone */ + int32 n_ci_sen; /**< number CI senones; these are the first */ + int32 n_sen; /**< number senones (CI+CD) */ + int32 n_tmat; /**< number transition matrices */ + + hash_table_t *ciphone_ht; /**< Hash table for mapping ciphone strings to ids */ + ciphone_t *ciphone; /**< CI-phone information for all ciphones */ + phone_t *phone; /**< Information for all ciphones and triphones */ + uint16 **sseq; /**< Unique state (or senone) sequences in this model, shared + among all phones/triphones */ + int32 n_sseq; /**< No. of unique senone sequences in this model */ + + int16 *cd2cisen; /**< Parent CI-senone id for each senone; the first + n_ci_sen are identity mappings; the CD-senones are + contiguous for each parent CI-phone */ + int16 *sen2cimap; /**< Parent CI-phone for each senone (CI or CD) */ + + int16 sil; /**< SILENCE_CIPHONE id */ + + ph_lc_t ***wpos_ci_lclist; /**< wpos_ci_lclist[wpos][ci] = list of lc for . + wpos_ci_lclist[wpos][ci][lc].rclist = list of rc for + . Only entries for the known triphones + are created to conserve space. + (NOTE: FOR INTERNAL USE ONLY.) */ +} mdef_t; + +/** Access macros; not meant for arbitrary use */ +#define mdef_is_fillerphone(m,p) ((m)->ciphone[p].filler) +#define mdef_n_ciphone(m) ((m)->n_ciphone) +#define mdef_n_phone(m) ((m)->n_phone) +#define mdef_n_sseq(m) ((m)->n_sseq) +#define mdef_n_emit_state(m) ((m)->n_emit_state) +#define mdef_n_sen(m) ((m)->n_sen) +#define mdef_n_tmat(m) ((m)->n_tmat) +#define mdef_pid2ssid(m,p) ((m)->phone[p].ssid) +#define mdef_pid2tmatid(m,p) ((m)->phone[p].tmat) +#define mdef_silphone(m) ((m)->sil) +#define mdef_sen2cimap(m) ((m)->sen2cimap) +#define mdef_sseq2sen(m,ss,pos) ((m)->sseq[ss][pos]) +#define mdef_pid2ci(m,p) ((m)->phone[p].ci) +#define mdef_cd2cisen(m) ((m)->cd2cisen) + +/** + * Initialize the phone structure from the given model definition file. + * It should be treated as a READ-ONLY structure. + * @return pointer to the phone structure created. + */ +mdef_t *mdef_init (char *mdeffile, /**< In: Model definition file */ + int breport /**< In: whether to report the progress or not */ + ); + + +/** + Get the ciphone id given a string name + @return ciphone id for the given ciphone string name +*/ +int mdef_ciphone_id(mdef_t *m, /**< In: Model structure being queried */ + char *ciphone /**< In: ciphone for which id wanted */ + ); + +/** + Get the phone string given the ci phone id. + @return: READ-ONLY ciphone string name for the given ciphone id +*/ +const char *mdef_ciphone_str(mdef_t *m, /**< In: Model structure being queried */ + int ci /**< In: ciphone id for which name wanted */ + ); + +/** + Decide whether the phone is ci phone. + @return 1 if given triphone argument is a ciphone, 0 if not, -1 if error +*/ +int mdef_is_ciphone (mdef_t *m, /**< In: Model structure being queried */ + int p /**< In: triphone id being queried */ + ); + +/** + Decide whether the senone is a senone for a ci phone, or a ci senone + @return 1 if a given senone is a ci senone +*/ +int mdef_is_cisenone(mdef_t *m, /**< In: Model structure being queried */ + int s /**< In: senone id being queried */ + ); + +/** + Decide the phone id given the left, right and base phones. + @return: phone id for the given constituents if found, else BAD_S3PID +*/ +int mdef_phone_id (mdef_t *m, /**< In: Model structure being queried */ + int b, /**< In: base ciphone id */ + int l, /**< In: left context ciphone id */ + int r, /**< In: right context ciphone id */ + word_posn_t pos /**< In: Word position */ + ); + +/** + * Create a phone string for the given phone (base or triphone) id in the given buf. + * @return 0 if successful, -1 if error. + */ +int mdef_phone_str(mdef_t *m, /**< In: Model structure being queried */ + int pid, /**< In: phone id being queried */ + char *buf /**< Out: On return, buf has the string */ + ); + +/** + * Compare the underlying HMMs for two given phones (i.e., compare the two transition + * matrix IDs and the individual state(senone) IDs). + * @return 0 iff the HMMs are identical, -1 otherwise. + */ +int mdef_hmm_cmp (mdef_t *m, /**< In: Model being queried */ + int p1, /**< In: One of the two triphones being compared */ + int p2 /**< In: One of the two triphones being compared */ + ); + +/** Report the model definition's parameters */ +void mdef_report(mdef_t *m /**< In: model definition structure */ + ); + +/** RAH, For freeing memory */ +void mdef_free_recursive_lc (ph_lc_t *lc /**< In: A list of left context */ + ); +void mdef_free_recursive_rc (ph_rc_t *rc /**< In: A list of right context */ + ); + +/** Free an mdef_t */ +void mdef_free (mdef_t *mdef /**< In : The model definition*/ + ); + + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/pocketsphinx_internal.h b/include/pocketsphinx_internal.h new file mode 100644 index 0000000..3f7dd98 --- /dev/null +++ b/include/pocketsphinx_internal.h @@ -0,0 +1,234 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 2008 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +/** + * @file pocketsphinx_internal.h Internal implementation of + * PocketSphinx decoder. + * @author David Huggins-Daines + */ + +#ifndef __POCKETSPHINX_INTERNAL_H__ +#define __POCKETSPHINX_INTERNAL_H__ + +/* SphinxBase headers. */ +#include +#include +#include +#include +#include +#include + +/* Local headers. */ +#include "pocketsphinx.h" +#include "acmod.h" +#include "dict.h" +#include "dict2pid.h" + +/** + * Search algorithm structure. + */ +typedef struct ps_search_s ps_search_t; + + +/* Search names*/ +#define PS_DEFAULT_SEARCH "_default" +#define PS_DEFAULT_PL_SEARCH "_default_pl" + +/* Search types */ +#define PS_SEARCH_TYPE_KWS "kws" +#define PS_SEARCH_TYPE_FSG "fsg" +#define PS_SEARCH_TYPE_NGRAM "ngram" +#define PS_SEARCH_TYPE_ALLPHONE "allphone" +#define PS_SEARCH_TYPE_STATE_ALIGN "state_align" +#define PS_SEARCH_TYPE_PHONE_LOOP "phone_loop" + +/** + * V-table for search algorithm. + */ +typedef struct ps_searchfuncs_s { + int (*start)(ps_search_t *search); + int (*step)(ps_search_t *search, int frame_idx); + int (*finish)(ps_search_t *search); + int (*reinit)(ps_search_t *search, dict_t *dict, dict2pid_t *d2p); + void (*free)(ps_search_t *search); + + ps_lattice_t *(*lattice)(ps_search_t *search); + char const *(*hyp)(ps_search_t *search, int32 *out_score); + int32 (*prob)(ps_search_t *search); + ps_seg_t *(*seg_iter)(ps_search_t *search); +} ps_searchfuncs_t; + +/** + * Base structure for search module. + */ +struct ps_search_s { + ps_searchfuncs_t *vt; /**< V-table of search methods. */ + + char *type; + char *name; + + ps_search_t *pls; /**< Phoneme loop for lookahead. */ + cmd_ln_t *config; /**< Configuration. */ + acmod_t *acmod; /**< Acoustic model. */ + dict_t *dict; /**< Pronunciation dictionary. */ + dict2pid_t *d2p; /**< Dictionary to senone mappings. */ + char *hyp_str; /**< Current hypothesis string. */ + ps_lattice_t *dag; /**< Current hypothesis word graph. */ + ps_latlink_t *last_link; /**< Final link in best path. */ + int32 post; /**< Utterance posterior probability. */ + int32 n_words; /**< Number of words known to search (may + be less than in the dictionary) */ + + /* Magical word IDs that must exist in the dictionary: */ + int32 start_wid; /**< Start word ID. */ + int32 silence_wid; /**< Silence word ID. */ + int32 finish_wid; /**< Finish word ID. */ +}; + +#define ps_search_base(s) ((ps_search_t *)s) +#define ps_search_config(s) ps_search_base(s)->config +#define ps_search_acmod(s) ps_search_base(s)->acmod +#define ps_search_dict(s) ps_search_base(s)->dict +#define ps_search_dict2pid(s) ps_search_base(s)->d2p +#define ps_search_dag(s) ps_search_base(s)->dag +#define ps_search_last_link(s) ps_search_base(s)->last_link +#define ps_search_post(s) ps_search_base(s)->post +#define ps_search_lookahead(s) ps_search_base(s)->pls +#define ps_search_n_words(s) ps_search_base(s)->n_words + +#define ps_search_type(s) ps_search_base(s)->type +#define ps_search_name(s) ps_search_base(s)->name +#define ps_search_start(s) (*(ps_search_base(s)->vt->start))(s) +#define ps_search_step(s,i) (*(ps_search_base(s)->vt->step))(s,i) +#define ps_search_finish(s) (*(ps_search_base(s)->vt->finish))(s) +#define ps_search_reinit(s,d,d2p) (*(ps_search_base(s)->vt->reinit))(s,d,d2p) +#define ps_search_free(s) (*(ps_search_base(s)->vt->free))(s) +#define ps_search_lattice(s) (*(ps_search_base(s)->vt->lattice))(s) +#define ps_search_hyp(s,sc) (*(ps_search_base(s)->vt->hyp))(s,sc) +#define ps_search_prob(s) (*(ps_search_base(s)->vt->prob))(s) +#define ps_search_seg_iter(s) (*(ps_search_base(s)->vt->seg_iter))(s) + +/* For convenience... */ +#define ps_search_silence_wid(s) ps_search_base(s)->silence_wid +#define ps_search_start_wid(s) ps_search_base(s)->start_wid +#define ps_search_finish_wid(s) ps_search_base(s)->finish_wid + +/** + * Initialize base structure. + */ +void ps_search_init(ps_search_t *search, ps_searchfuncs_t *vt, + const char *type, const char *name, + cmd_ln_t *config, acmod_t *acmod, dict_t *dict, + dict2pid_t *d2p); + + +/** + * Free search + */ +void ps_search_base_free(ps_search_t *search); + +/** + * Re-initialize base structure with new dictionary. + */ +void ps_search_base_reinit(ps_search_t *search, dict_t *dict, + dict2pid_t *d2p); + +typedef struct ps_segfuncs_s { + ps_seg_t *(*seg_next)(ps_seg_t *seg); + void (*seg_free)(ps_seg_t *seg); +} ps_segfuncs_t; + +/** + * Base structure for hypothesis segmentation iterator. + */ +struct ps_seg_s { + ps_segfuncs_t *vt; /**< V-table of seg methods */ + ps_search_t *search; /**< Search object from whence this came */ + char const *word; /**< Word string (pointer into dictionary hash) */ + frame_idx_t sf; /**< Start frame. */ + frame_idx_t ef; /**< End frame. */ + int32 ascr; /**< Acoustic score. */ + int32 lscr; /**< Language model score. */ + int32 prob; /**< Log posterior probability. */ + /* This doesn't need to be 32 bits, so once the scores above are + * reduced to 16 bits (or less!), this will be too. */ + int32 lback; /**< Language model backoff. */ + /* Not sure if this should be here at all. */ + float32 lwf; /**< Language weight factor (for second-pass searches) */ +}; + +#define ps_search_seg_next(seg) (*(seg->vt->seg_next))(seg) +#define ps_search_seg_free(s) (*(seg->vt->seg_free))(seg) + + +/** + * Decoder object. + */ +struct ps_decoder_s { + /* Model parameters and such. */ + cmd_ln_t *config; /**< Configuration. */ + int refcount; /**< Reference count. */ + + /* Basic units of computation. */ + acmod_t *acmod; /**< Acoustic model. */ + dict_t *dict; /**< Pronunciation dictionary. */ + dict2pid_t *d2p; /**< Dictionary to senone mapping. */ + logmath_t *lmath; /**< Log math computation. */ + + /* Search modules. */ + hash_table_t *searches; /**< Set of search modules. */ + /* TODO: Convert this to a stack of searches each with their own + * lookahead value. */ + ps_search_t *search; /**< Currently active search module. */ + ps_search_t *phone_loop; /**< Phone loop search for lookahead. */ + int pl_window; /**< Window size for phoneme lookahead. */ + + /* Utterance-processing related stuff. */ + uint32 uttno; /**< Utterance counter. */ + ptmr_t perf; /**< Performance counter for all of decoding. */ + uint32 n_frame; /**< Total number of frames processed. */ + char const *mfclogdir; /**< Log directory for MFCC files. */ + char const *rawlogdir; /**< Log directory for audio files. */ + char const *senlogdir; /**< Log directory for senone score files. */ +}; + + +struct ps_search_iter_s { + hash_iter_t itor; +}; + +#endif /* __POCKETSPHINX_INTERNAL_H__ */ diff --git a/include/ps_alignment.h b/include/ps_alignment.h new file mode 100644 index 0000000..4774bef --- /dev/null +++ b/include/ps_alignment.h @@ -0,0 +1,190 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 2010 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +/** + * @file ps_alignment.h Multi-level alignment structure + */ + +#ifndef __PS_ALIGNMENT_H__ +#define __PS_ALIGNMENT_H__ + +/* System headers. */ + +/* SphinxBase headers. */ +#include + +/* Local headers. */ +#include "dict2pid.h" +#include "hmm.h" + +#define PS_ALIGNMENT_NONE ((uint16)0xffff) + +struct ps_alignment_entry_s { + union { + int32 wid; + struct { + uint16 ssid; + uint16 cipid; + uint16 tmatid; + } pid; + uint16 senid; + } id; + int16 start; + int16 duration; + int32 score; + uint16 parent; + uint16 child; +}; +typedef struct ps_alignment_entry_s ps_alignment_entry_t; + +struct ps_alignment_vector_s { + ps_alignment_entry_t *seq; + uint16 n_ent, n_alloc; +}; +typedef struct ps_alignment_vector_s ps_alignment_vector_t; + +struct ps_alignment_s { + dict2pid_t *d2p; + ps_alignment_vector_t word; + ps_alignment_vector_t sseq; + ps_alignment_vector_t state; +}; +typedef struct ps_alignment_s ps_alignment_t; + +struct ps_alignment_iter_s { + ps_alignment_t *al; + ps_alignment_vector_t *vec; + int pos; +}; +typedef struct ps_alignment_iter_s ps_alignment_iter_t; + +/** + * Create a new, empty alignment. + */ +ps_alignment_t *ps_alignment_init(dict2pid_t *d2p); + +/** + * Release an alignment + */ +int ps_alignment_free(ps_alignment_t *al); + +/** + * Append a word. + */ +int ps_alignment_add_word(ps_alignment_t *al, + int32 wid, int duration); + +/** + * Populate lower layers using available word information. + */ +int ps_alignment_populate(ps_alignment_t *al); + +/** + * Populate lower layers using context-independent phones. + */ +int ps_alignment_populate_ci(ps_alignment_t *al); + +/** + * Propagate timing information up from state sequence. + */ +int ps_alignment_propagate(ps_alignment_t *al); + +/** + * Number of words. + */ +int ps_alignment_n_words(ps_alignment_t *al); + +/** + * Number of phones. + */ +int ps_alignment_n_phones(ps_alignment_t *al); + +/** + * Number of states. + */ +int ps_alignment_n_states(ps_alignment_t *al); + +/** + * Iterate over the alignment starting at the first word. + */ +ps_alignment_iter_t *ps_alignment_words(ps_alignment_t *al); + +/** + * Iterate over the alignment starting at the first phone. + */ +ps_alignment_iter_t *ps_alignment_phones(ps_alignment_t *al); + +/** + * Iterate over the alignment starting at the first state. + */ +ps_alignment_iter_t *ps_alignment_states(ps_alignment_t *al); + +/** + * Get the alignment entry pointed to by an iterator. + */ +ps_alignment_entry_t *ps_alignment_iter_get(ps_alignment_iter_t *itor); + +/** + * Move alignment iterator to given index. + */ +ps_alignment_iter_t *ps_alignment_iter_goto(ps_alignment_iter_t *itor, int pos); + +/** + * Move an alignment iterator forward. + */ +ps_alignment_iter_t *ps_alignment_iter_next(ps_alignment_iter_t *itor); + +/** + * Move an alignment iterator back. + */ +ps_alignment_iter_t *ps_alignment_iter_prev(ps_alignment_iter_t *itor); + +/** + * Get a new iterator starting at the parent of the current node. + */ +ps_alignment_iter_t *ps_alignment_iter_up(ps_alignment_iter_t *itor); +/** + * Get a new iterator starting at the first child of the current node. + */ +ps_alignment_iter_t *ps_alignment_iter_down(ps_alignment_iter_t *itor); + +/** + * Release an iterator before completing all iterations. + */ +int ps_alignment_iter_free(ps_alignment_iter_t *itor); + +#endif /* __PS_ALIGNMENT_H__ */ diff --git a/include/s3types.h b/include/s3types.h new file mode 100644 index 0000000..b40f4c1 --- /dev/null +++ b/include/s3types.h @@ -0,0 +1,99 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 1999-2004 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +#ifndef _S3_S3TYPES_H_ +#define _S3_S3TYPES_H_ + +#include +#include + +#include +#include +#include + +/** \file s3types.h + * \brief Size definition of semantically units. Common for both s3 and s3.X decoder. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Size definitions for more semantially meaningful units. + * Illegal value definitions, limits, and tests for specific types. + * NOTE: Types will be either int32 or smaller; only smaller ones may be unsigned (i.e., + * no type will be uint32). + */ + +typedef int16 s3cipid_t; /** Ci phone id */ +#define BAD_S3CIPID ((s3cipid_t) -1) +#define NOT_S3CIPID(p) ((p)<0) +#define IS_S3CIPID(p) ((p)>=0) +#define MAX_S3CIPID 32767 + +/*#define MAX_S3CIPID 127*/ + +typedef int32 s3pid_t; /** Phone id (triphone or ciphone) */ +#define BAD_S3PID ((s3pid_t) -1) +#define NOT_S3PID(p) ((p)<0) +#define IS_S3PID(p) ((p)>=0) +#define MAX_S3PID ((int32)0x7ffffffe) + +typedef uint16 s3ssid_t; /** Senone sequence id (triphone or ciphone) */ +#define BAD_S3SSID ((s3ssid_t) 0xffff) +#define NOT_S3SSID(p) ((p) == BAD_S3SSID) +#define IS_S3SSID(p) ((p) != BAD_S3SSID) +#define MAX_S3SSID ((s3ssid_t)0xfffe) + +typedef int32 s3tmatid_t; /** Transition matrix id; there can be as many as pids */ +#define BAD_S3TMATID ((s3tmatid_t) -1) +#define NOT_S3TMATID(t) ((t)<0) +#define IS_S3TMATID(t) ((t)>=0) +#define MAX_S3TMATID ((int32)0x7ffffffe) + +typedef int32 s3wid_t; /** Dictionary word id */ +#define BAD_S3WID ((s3wid_t) -1) +#define NOT_S3WID(w) ((w)<0) +#define IS_S3WID(w) ((w)>=0) +#define MAX_S3WID ((int32)0x7ffffffe) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/state_align_search.h b/include/state_align_search.h new file mode 100644 index 0000000..af102a6 --- /dev/null +++ b/include/state_align_search.h @@ -0,0 +1,87 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 2010 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +/** + * @file state_align_search.h State (and phone and word) alignment search. + */ + +#ifndef __STATE_ALIGN_SEARCH_H__ +#define __STATE_ALIGN_SEARCH_H__ + +/* SphinxBase headers. */ +#include + +/* Local headers. */ +#include "pocketsphinx_internal.h" +#include "ps_alignment.h" +#include "hmm.h" + + +/** + * History structure + */ +struct state_align_hist_s { + uint16 id; + int32 score; +}; +typedef struct state_align_hist_s state_align_hist_t; + +/** + * Phone loop search structure. + */ +struct state_align_search_s { + ps_search_t base; /**< Base search structure. */ + hmm_context_t *hmmctx; /**< HMM context structure. */ + ps_alignment_t *al; /**< Alignment structure being operated on. */ + hmm_t *hmms; /**< Vector of HMMs corresponding to phone level. */ + int n_phones; /**< Number of HMMs (phones). */ + + int frame; /**< Current frame being processed. */ + int32 best_score; /**< Best score in current frame. */ + + int n_emit_state; /**< Number of emitting states (tokens per frame) */ + state_align_hist_t *tokens; /**< Tokens (backpointers) for state alignment. */ + int n_fr_alloc; /**< Number of frames of tokens allocated. */ +}; +typedef struct state_align_search_s state_align_search_t; + +ps_search_t *state_align_search_init(const char *name, + cmd_ln_t *config, + acmod_t *acmod, + ps_alignment_t *al); + +#endif /* __STATE_ALIGN_SEARCH_H__ */ diff --git a/include/tmat.h b/include/tmat.h new file mode 100644 index 0000000..5b64211 --- /dev/null +++ b/include/tmat.h @@ -0,0 +1,98 @@ +/* ==================================================================== + * Copyright (c) 1999-2004 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +#ifndef _S3_TMAT_H_ +#define _S3_TMAT_H_ + +#include +#include + +/** \file tmat.h + * \brief Transition matrix data structure. + */ +#ifdef __cplusplus +extern "C" { +#endif + +/** + * \struct tmat_t + * \brief Transition matrix data structure. All phone HMMs are assumed to have the same + * topology. + */ +typedef struct { + uint8 ***tp; /**< The transition matrices; kept in the same scale as acoustic scores; + tp[tmatid][from-state][to-state] */ + int16 n_tmat; /**< Number matrices */ + int16 n_state; /**< Number source states in matrix (only the emitting states); + Number destination states = n_state+1, it includes the exit state */ +} tmat_t; + + +/** Initialize transition matrix */ + +tmat_t *tmat_init (char const *tmatfile,/**< In: input file */ + logmath_t *lmath, /**< In: log math parameters */ + float64 tpfloor, /**< In: floor value for each non-zero transition probability */ + int32 breport /**< In: whether reporting the process of tmat_t */ + ); + + + +/** Dumping the transition matrix for debugging */ + +void tmat_dump (tmat_t *tmat, /**< In: transition matrix */ + FILE *fp /**< In: file pointer */ + ); + + +/** + * RAH, add code to remove memory allocated by tmat_init + */ + +void tmat_free (tmat_t *t /**< In: transition matrix */ + ); + +/** + * Report the detail of the transition matrix structure. + */ +void tmat_report(tmat_t *t /**< In: transition matrix*/ + ); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 0000000..ce0130c --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add all the src cpp files with the headers +add_executable(featex.o featex.c) + +# Ensure everything in `include` is included. +target_include_directories(featex.o PUBLIC ../include) +target_include_directories(featex.o PUBLIC "/usr/local/include/pocketsphinx") +target_include_directories(featex.o PUBLIC "/usr/local/include/sphinxbase") + +target_link_libraries(featex.o pocketsphinx sphinxbase m) + +# IDEs should put the headers in a nice place +source_group(TREE "${PROJECT_SOURCE_DIR}/include" PREFIX "Header Files" FILES) diff --git a/src/featex.c b/src/featex.c new file mode 100644 index 0000000..65e9ae7 --- /dev/null +++ b/src/featex.c @@ -0,0 +1,615 @@ +#define FRATE 65 +#define MODELDIR "/usr/local/share/pocketsphinx/model/en-us/en-us" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ps_alignment.h" +#include +#include "state_align_search.h" +#include "pocketsphinx_internal.h" +#include "ps_search.h" + +const char *argp_program_version = + "Featex 0.1"; + +const char *argp_program_bug_address = + ""; + +/* This structure is used by main to communicate with parse_opt. */ +struct arguments +{ + char *COMBO; + char *INFILE; + int PLAY; + char *PHRASE; +}; + +/* + OPTIONS. Field 1 in ARGP. + Order of fields: {NAME, KEY, ARG, FLAGS, DOC}. +*/ +static struct argp_option options[] = + { + {"combo", 'c', "COMBO_PATH", 0, "Path to combo.dict."}, + {"infile", 'i', "INFILE_PATH", 0, "Path to input raw file."}, + {"utterance", 'u', 0, 0, "Toggle play utterances"}, + {"word", 'w', 0, 0, "Toggle play words"}, + {"phonemes", 'p', 0, 0, "Toggle play phonemes"}, + {"triphones", 't', 0, 0, "Toggle play triphones"}, + {"diphones", 'd', 0, 0, "Toggle play diphones"}, + {"phrase", 'P', "'PHRASE'", 0, "Input phrase"}, + {0}}; + +/* + PARSER. Field 2 in ARGP. + Order of parameters: KEY, ARG, STATE. +*/ +static error_t +parse_opt(int key, char *arg, struct argp_state *state) +{ + struct arguments *arguments = state->input; + + switch (key) + { + case 'c': + arguments->COMBO = arg; + break; + case 'i': + arguments->INFILE = arg; + break; + case 'P': + arguments->PHRASE = arg; + break; + case 'u': + arguments->PLAY |= 1; + break; + case 'w': + arguments->PLAY |= 2; + break; + case 'p': + arguments->PLAY |= 4; + break; + case 't': + arguments->PLAY |= 8; + break; + case 'd': + arguments->PLAY |= 16; + break; + + default: + return ARGP_ERR_UNKNOWN; + } + return 0; +} + +/* + DOC. Field 4 in ARGP. + Program documentation. +*/ +static char doc[] = + "featex -- PocketSphinx phonetic feature extraction for intelligibility prediction and remediation"; + +/* + The ARGP structure itself. +*/ +static struct argp argp = {options, parse_opt, 0, doc}; + +int main(int argc, char **argv) +{ + + struct arguments arguments; + + /* Set argument defaults */ + arguments.COMBO = "combo.dict"; + arguments.INFILE = "featex.raw"; + arguments.PHRASE = ""; + arguments.PLAY = 0; + + /* Where the magic happens */ + argp_parse(&argp, argc, argv, 0, 0, &arguments); + + if (arguments.PHRASE == "") + { + fprintf(stderr, "Missing phrase, ensure you used -P\n"); + exit(-1); + } + + /* Debug - Print argument values */ + // printf("alpha = %s\nbravo = %s\nPLAY = %i\nPHRASE = %s", + // arguments.COMBO, arguments.INFILE, arguments.PLAY, arguments.PHRASE); + + // exit(0); + + ps_decoder_t *ps; + dict_t *dict; + dict2pid_t *d2p; + acmod_t *acmod; + bin_mdef_t *mdef; + ps_alignment_t *al; + ps_alignment_iter_t *itor, *itor2; + ps_search_t *search; + cmd_ln_t *config; + FILE *rawfh; + int16 buf[2048]; + char *fbuf, *fbip, *obuf; + size_t nread; + int16 const *bptr; + int sz, nfr, wend, n, maxdur, i, j, k, found; + int play = arguments.PLAY; + ps_alignment_entry_t *ae; + char grammar[1000], target[10], frates[10]; + char *p, *q, *r; // string manipulation pointers for constructing grammar + ps_nbest_t *nb; + int32 score; + hash_table_t *hyptbl; + double frated; + + struct + { + int start, dur, cipid, score; + } * algn; + +#define FPS (16000 / FRATE * 2) + + sprintf(frates, "%d", FRATE); + frated = (double)FRATE; + config = cmd_ln_init(NULL, ps_args(), FALSE, + "-hmm", MODELDIR, + "-dict", arguments.COMBO, + "-samprate", "16000", + "-topn", "64", // TODO parameterize for proper optimization + "-beam", "1e-57", + "-wbeam", "1e-56", + "-maxhmmpf", "-1", + "-frate", frates, + "-fsgusefiller", "no", + NULL); + if (!(ps = ps_init(config))) + { + fprintf(stderr, "%s: ps_init() failed.\n", argv[0]); + return 2; + } + dict = ps->dict; + d2p = ps->d2p; + acmod = ps->acmod; + mdef = acmod->mdef; + + al = ps_alignment_init(d2p); + ps_alignment_add_word(al, dict_wordid(dict, ""), 0); + + char words[50][50]; + int letter, cnt; + letter = cnt = 0; + for (i = 0; i < strlen(arguments.PHRASE); i++) + { + if (arguments.PHRASE[i] == ' ' || arguments.PHRASE[i] == '\0') + { + words[cnt][letter] = '\0'; + cnt++; //for next word + letter = 0; //for next word, init index to 0 + } + else + { + words[cnt][letter] = arguments.PHRASE[i]; + letter++; + } + } + + for (int i = 0; i < cnt; i++) + { + n = dict_wordid(dict, words[i]); + if (n < 0) + { + fprintf(stderr, "%s: unrecogized word: %s\n", argv[0], words[i]); + return 3; + } + ps_alignment_add_word(al, n, 0); + } + + ps_alignment_add_word(al, dict_wordid(dict, ""), 0); + ps_alignment_populate(al); + + search = state_align_search_init("state_align", config, acmod, al); + + rawfh = fopen(arguments.INFILE, "rb"); + if (!rawfh) + { + fprintf(stderr, "%s: can't open audio input file: %s\n", + argv[0], arguments.INFILE); + return 4; + } + fseek(rawfh, 0L, SEEK_END); + sz = ftell(rawfh); + fbuf = fbip = malloc(sz); + rewind(rawfh); + while (!feof(rawfh)) + { + nread = fread(buf, sizeof(*buf), 2048, rawfh); + memcpy(fbip, buf, nread * sizeof(*buf)); + fbip += nread * sizeof(*buf); + } + + acmod_start_utt(acmod); + ps_search_start(search); + + bptr = (const int16 *)fbuf; + nread = (fbip - fbuf) / sizeof(*buf); + while ((nfr = acmod_process_raw(acmod, &bptr, &nread, TRUE)) > 0) + { + while (acmod->n_feat_frame > 0) + { + ps_search_step(search, acmod->output_frame); + acmod_advance(acmod); + } + fprintf(stderr, "%s: processed %d frames\n", argv[0], nfr); + } + + acmod_end_utt(acmod); + ps_search_finish(search); + + fprintf(stderr, "%s: aligned %d words, %d phones, and %d states\n", + argv[0], ps_alignment_n_words(al), ps_alignment_n_phones(al), + ps_alignment_n_states(al)); + + if (play & 1) + { // play utterance + rawfh = fopen("/tmp/outphone.raw", "wb"); + fwrite(fbuf, sz, 1, rawfh); + fclose(rawfh); + system("play -q -r16k -ts16 -c1 /tmp/outphone.raw"); + remove("/tmp/outphone.raw"); + } + + algn = malloc(sizeof(*algn) * ps_alignment_n_phones(al)); + obuf = malloc(8000); + memset(obuf, 0, 8000); + n = 0; + + maxdur = 0; + + itor = ps_alignment_words(al); + while (itor) + { + ae = ps_alignment_iter_get(itor); + fprintf(stderr, "%s: word '%s': %.2fs for %.2fs, score %d\n", argv[0], + dict->word[ae->id.wid].word, ae->start / frated, + ae->duration / frated, ae->score); + if (play & 2) + { // play words + rawfh = fopen("/tmp/outphone.raw", "wb"); + fwrite(obuf, 8000, 1, rawfh); + fwrite(fbuf + ae->start * FPS, ae->duration * FPS, 1, rawfh); + fwrite(obuf, 8000, 1, rawfh); + fclose(rawfh); + system("play -q -r16k -ts16 -c1 /tmp/outphone.raw"); + remove("/tmp/outphone.raw"); + } + itor2 = ps_alignment_iter_down(itor); + wend = ae->duration + ae->start; + while (itor2) + { + ae = ps_alignment_iter_get(itor2); + if (ae->start >= wend) + break; + fprintf(stderr, "%s: sub-phone '%s': %.2fs for %.2fs, score %d\n", + argv[0], mdef->ciname[ae->id.pid.cipid], ae->start / frated, + ae->duration / frated, ae->score); + algn[n].start = ae->start; + algn[n].dur = ae->duration; + algn[n].score = ae->score; + algn[n++].cipid = ae->id.pid.cipid; + if (ae->duration > maxdur) + maxdur = ae->duration; + itor2 = ps_alignment_iter_next(itor2); + } + itor = ps_alignment_iter_next(itor); + } + + ps_search_free(search); + ps_alignment_free(al); + free(obuf); + + obuf = malloc(16000 + FPS * maxdur * 3); + memset(obuf, 0, 8000); + for (i = 0; i < n; i++) + { + memcpy(obuf + 8000, fbuf + algn[i].start * FPS, algn[i].dur * FPS); + memset(obuf + 8000 + algn[i].dur * FPS, 0, 8000); + + fprintf(stderr, "%s: phoneme %d: %s %.2fs for %.2fs, score %d\n", + argv[0], i + 1, mdef->ciname[algn[i].cipid], + algn[i].start / frated, algn[i].dur / frated, algn[i].score); + if (play & 4) + { // play phonemes (default if '-p' specified) + rawfh = fopen("/tmp/outphone.raw", "wb"); + fwrite(obuf, 16000 + algn[i].dur * FPS, 1, rawfh); + fclose(rawfh); + system("play -q -r16k -ts16 -c1 /tmp/outphone.raw"); + remove("/tmp/outphone.raw"); + } + } + + hyptbl = hash_table_new(175, HASH_CASE_YES); // for hypothesis deduplication + + for (i = 1; i < n; i++) + { + + if (i == n - 1) + goto lastdiphone; + + if (i > 1) + printf(" "); + + printf("%.2f %.3f", algn[i].dur / frated, 1 / log(2 - algn[i].score)); + + memcpy(obuf + 8000, fbuf + algn[i - 1].start * FPS, + (algn[i - 1].dur + algn[i].dur + algn[i + 1].dur) * FPS); + memset(obuf + 8000 + (algn[i - 1].dur + algn[i].dur + algn[i + 1].dur) * FPS, + 0, 8000); + + fprintf(stderr, "%s: triphone %d: %s-%s-%s\n", argv[0], i, + mdef->ciname[algn[i - 1].cipid], + mdef->ciname[algn[i].cipid], + mdef->ciname[algn[i + 1].cipid]); + if (play & 8) + { // play triphones + rawfh = fopen("/tmp/outphone.raw", "wb"); + fwrite(obuf, 16000 + (algn[i - 1].dur + algn[i].dur + algn[i + 1].dur) * FPS, 1, rawfh); + fclose(rawfh); + system("play -q -r16k -ts16 -c1 /tmp/outphone.raw"); + remove("/tmp/outphone.raw"); + } + + grammar[0] = '\0'; + strcat(grammar, "#JSGF V1.0;\ngrammar subalts;\npublic = sil1 "); + if (algn[i - 1].cipid != mdef->sil) + { + p = mdef->ciname[algn[i - 1].cipid]; + q = grammar; + while (*++q) + ; + while (*p) + *q++ = tolower(*p++); + *q++ = '2'; + *q = '\0'; + } + strcat(grammar, " [ aa3 | ae3 | ah3 | ao3 | aw3 | ay3 | b3 | ch3 | d3" + " | dh3 | eh3 | er3 | ey3 | f3 | g3 | hh3 | ih3 | iy3 | jh3" + " | k3 | l3 | m3 | n3 | ng3 | ow3 | oy3 | p3 | r3 | s3 | sh3" + " | sil3 | t3 | th3 | uh3 | uw3 | v3 | w3 | y3 | z3 | zh3 ] "); + if (algn[i + 1].cipid != mdef->sil) + { + p = mdef->ciname[algn[i + 1].cipid]; + q = grammar; + while (*++q) + ; + while (*p) + *q++ = tolower(*p++); + *q++ = '4'; + *q = '\0'; + } + strcat(grammar, " sil5 ;\n"); + + fprintf(stderr, "%s: %s", argv[0], grammar); + + ps_set_jsgf_string(ps, "subalts", grammar); + ps_set_search(ps, "subalts"); + ps_start_utt(ps); + ps_process_raw(ps, (const int16 *)obuf, 8000 + // samples not bytes + (algn[i - 1].dur + algn[i].dur + algn[i + 1].dur) * 160, + FALSE, TRUE); + ps_end_utt(ps); + + nb = ps_nbest(ps); + j = found = 0; + target[0] = ' '; + target[1] = '\0'; + strcat(target, mdef->ciname[algn[i].cipid]); + strcat(target, "3"); + p = target; + while (*++p) + { + *p = tolower(*p); + } + while (nb) + { + p = (char *)ps_nbest_hyp(nb, &score); + if (p) + { // some hypotheses are literally NULL + q = p; + while (*++q) + ; + if (*(q - 1) == '5') + { // ignore hypotheses w/o whole match + + // ignore repeated hypotheses + if (hash_table_lookup(hyptbl, p, NULL) == -1) + { + j++; + fprintf(stderr, "%s: triphone hypothesis %d: %s, %d\n", + argv[0], j, p, score); + hash_table_enter_int32(hyptbl, p, score); + + if (strstr(p, target)) + { + found++; + ps_nbest_free(nb); + break; + } + } + } + } + nb = ps_nbest_next(nb); + } + if (!found) + j = 42; // zero for bad recognition results or no match + fprintf(stderr, "%s: SUBSTITUTION: %.3f\n", argv[0], (42.0 - j) / 42.0); + printf(" %.3f", (42.0 - j) / 42.0); + + hash_table_empty(hyptbl); + + lastdiphone: // goto target for the final set of two phonemes + + memcpy(obuf + 8000, fbuf + algn[i - 1].start * FPS, + (algn[i - 1].dur + algn[i].dur) * FPS); + memset(obuf + 8000 + (algn[i - 1].dur + algn[i].dur) * FPS, 0, 8000); + + fprintf(stderr, "%s: diphone %d: %s-%s\n", argv[0], i, + mdef->ciname[algn[i - 1].cipid], + mdef->ciname[algn[i].cipid]); + if (play & 16) + { // play diphones + rawfh = fopen("/tmp/outphone.raw", "wb"); + fwrite(obuf, 16000 + (algn[i - 1].dur + algn[i].dur) * FPS, 1, rawfh); + fclose(rawfh); + system("play -q -r16k -ts16 -c1 /tmp/outphone.raw"); + remove("/tmp/outphone.raw"); + } + + grammar[0] = '\0'; + strcat(grammar, + "#JSGF V1.0;\ngrammar insdels;\npublic = sil1 [ "); + p = mdef->ciname[algn[i - 1].cipid]; + q = grammar; + while (*++q) + ; + while (*p) + *q++ = tolower(*p++); + *q++ = '2'; + *q = '\0'; + r = q; + strcat(grammar, " ] [ aa3| ae3 | ah3 | ao3 | aw3 | ay3 | b3 | ch3" + " | d3 | dh3 | eh3 | er3 | ey3 | f3 | g3 | hh3 | ih3 | iy3" + " | jh3 | k3 | l3 | m3 | n3 | ng3 | ow3 | oy3 | p3 | r3 " + " | s3 | sh3 | sil3 | t3 | th3 | uh3 | uw3 | v3 | w3 | y3 " + " | z3 | zh3 ] "); + p = mdef->ciname[algn[i - 1].cipid]; // first in diphone + while (*++q) + { // blank out expected phoneme from possible insertions + if (isalpha(*q)) + { + if ((*q == tolower(*p)) && (((*(q + 1) == '3') && *(p + 1) == '\0') || *(q + 1) == tolower(*(p + 1)))) + { + *(q - 2) = ' '; // blank out preceding '|' + *q = ' '; + *(q + 1) = ' '; + *(q + 2) = ' '; + *(q + 3) = ' '; + } + else + { + q += 3; // advance past the rest of the phoneme + } + } + } + p = mdef->ciname[algn[i].cipid]; // second in diphone + q = r; + while (*++q) + { // blank out expected phoneme from possible insertions + if (isalpha(*q)) + { + if ((*q == tolower(*p)) && (((*(q + 1) == '3') && *(p + 1) == '\0') || *(q + 1) == tolower(*(p + 1)))) + { + *(q - 2) = ' '; // blank out preceding '|' + *q = ' '; + *(q + 1) = ' '; + *(q + 2) = ' '; + *(q + 3) = ' '; + } + else + { + q += 3; // advance past the rest of the phoneme + } + } + } + if (algn[i].cipid != mdef->sil) + { + p = mdef->ciname[algn[i].cipid]; + q = grammar; + while (*++q) + ; + while (*p) + *q++ = tolower(*p++); + *q++ = '4'; + *q = '\0'; + } + strcat(grammar, " sil5 ;\n"); + + fprintf(stderr, "%s: %s", argv[0], grammar); + + ps_set_jsgf_string(ps, "insdels", grammar); + ps_set_search(ps, "insdels"); + ps_start_utt(ps); + ps_process_raw(ps, (const int16 *)obuf, 8000 + // samples not bytes + (algn[i - 1].dur + algn[i].dur) * 160, + FALSE, TRUE); + ps_end_utt(ps); + + nb = ps_nbest(ps); + j = k = found = 0; + while (nb) + { + p = (char *)ps_nbest_hyp(nb, &score); + if (p) + { // some hypotheses are literally NULL + q = p; + while (*++q) + ; + if (*(q - 1) == '5') + { // ignore hypotheses w/o whole match + + // ignore repeated hypotheses + if (hash_table_lookup(hyptbl, p, NULL) == -1) + { + j++; + fprintf(stderr, "%s: diphone hypothesis %d: %s, %d\n", + argv[0], j, p, score); + hash_table_enter_int32(hyptbl, p, score); + + if (!strstr(p, "2 ")) + k++; + if (strstr(p, "3 ")) + k++; + + if (strstr(p, "2 ") && !strstr(p, "3 ")) + { + found++; + ps_nbest_free(nb); + break; + } + } + } + } + nb = ps_nbest_next(nb); + } + if (j == 0) + k = 160; // zero for bad recognition results + else if (!found) + { + k += 80; // add half the range if the preferred hypothesis missed + if (k > 160) + k = 160; // clamp + } + fprintf(stderr, "%s: INS/DEL: %.3f\n", argv[0], (160.0 - k) / 160); + printf(" %.3f", (160.0 - k) / 160.0); + + hash_table_empty(hyptbl); + } + + printf("\n"); + + free(obuf); + free(algn); + free(fbuf); + ps_free(ps); + cmd_ln_free_r(config); + + return 0; +}