From 9eef27005ec4064795de3e66d2181218d4448a1a Mon Sep 17 00:00:00 2001 From: Joshua Arulsamy Date: Tue, 26 May 2020 17:00:07 -0600 Subject: [PATCH 1/9] Added gitignore --- .gitignore | 129 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..48ab6b7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,129 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don’t work, or not +# install all needed dependencies. +#Pipfile.lock + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# Vscode +.vscode/ + +# Binaries +tic.out From bc210f0eab1b8e2ed9a90d2a53f482a8f82ca42a Mon Sep 17 00:00:00 2001 From: Joshua Arulsamy Date: Tue, 26 May 2020 17:00:17 -0600 Subject: [PATCH 2/9] Switched to cmake for building The old way of building was difficult and cumbersome. This way, there is no need to link against other projects src. Added all the necessary headers that aren't installed by other libs. --- CMakeLists.txt | 22 ++ include/acmod.h | 466 ++++++++++++++++++++++++++++++++ include/allphone_search.h | 179 ++++++++++++ include/bin_mdef.h | 236 ++++++++++++++++ include/blkarray_list.h | 139 ++++++++++ include/dict.h | 210 ++++++++++++++ include/dict2pid.h | 180 ++++++++++++ include/fsg_history.h | 215 +++++++++++++++ include/fsg_lextree.h | 255 +++++++++++++++++ include/fsg_search_internal.h | 153 +++++++++++ include/hmm.h | 306 +++++++++++++++++++++ include/kws_detections.h | 76 ++++++ include/kws_search.h | 142 ++++++++++ include/mdef.h | 271 +++++++++++++++++++ include/ms_gauden.h | 150 ++++++++++ include/ms_mgau.h | 143 ++++++++++ include/ms_senone.h | 131 +++++++++ include/ngram_search.h | 434 +++++++++++++++++++++++++++++ include/ngram_search_fwdflat.h | 81 ++++++ include/ngram_search_fwdtree.h | 83 ++++++ include/phone_loop_search.h | 102 +++++++ include/pocketsphinx_internal.h | 234 ++++++++++++++++ include/ps_alignment.h | 190 +++++++++++++ include/ps_lattice_internal.h | 282 +++++++++++++++++++ include/ptm_mgau.h | 103 +++++++ include/s2_semi_mgau.h | 98 +++++++ include/s3types.h | 99 +++++++ include/state_align_search.h | 87 ++++++ include/tied_mgau_common.h | 121 +++++++++ include/tmat.h | 98 +++++++ include/vector.h | 89 ++++++ src/CMakeLists.txt | 12 + featex.c => src/featex.c | 391 ++++++++++++++++----------- 33 files changed, 5621 insertions(+), 157 deletions(-) create mode 100644 CMakeLists.txt create mode 100644 include/acmod.h create mode 100644 include/allphone_search.h create mode 100644 include/bin_mdef.h create mode 100644 include/blkarray_list.h create mode 100644 include/dict.h create mode 100644 include/dict2pid.h create mode 100644 include/fsg_history.h create mode 100644 include/fsg_lextree.h create mode 100644 include/fsg_search_internal.h create mode 100644 include/hmm.h create mode 100644 include/kws_detections.h create mode 100644 include/kws_search.h create mode 100644 include/mdef.h create mode 100644 include/ms_gauden.h create mode 100644 include/ms_mgau.h create mode 100644 include/ms_senone.h create mode 100644 include/ngram_search.h create mode 100644 include/ngram_search_fwdflat.h create mode 100644 include/ngram_search_fwdtree.h create mode 100644 include/phone_loop_search.h create mode 100644 include/pocketsphinx_internal.h create mode 100644 include/ps_alignment.h create mode 100644 include/ps_lattice_internal.h create mode 100644 include/ptm_mgau.h create mode 100644 include/s2_semi_mgau.h create mode 100644 include/s3types.h create mode 100644 include/state_align_search.h create mode 100644 include/tied_mgau_common.h create mode 100644 include/tmat.h create mode 100644 include/vector.h create mode 100644 src/CMakeLists.txt rename featex.c => src/featex.c (51%) diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..bc83780 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,22 @@ +# Works with 3.11 and tested through 3.15 +cmake_minimum_required(VERSION 3.11...3.16) + +# Project name and a few useful settings. Other commands can pick up the results +project(featex + VERSION 0.1 + DESCRIPTION "featex" + LANGUAGES C) + +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + +# Only do these if this is the main project, and not if it is included through add_subdirectory +if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) + + # Nicely support folders in IDE's + set_property(GLOBAL PROPERTY USE_FOLDERS ON) + +endif() + +add_subdirectory(src) diff --git a/include/acmod.h b/include/acmod.h new file mode 100644 index 0000000..f4d5761 --- /dev/null +++ b/include/acmod.h @@ -0,0 +1,466 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 2008 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +/** + * @file acmod.h Acoustic model structures for PocketSphinx. + * @author David Huggins-Daines + */ + +#ifndef __ACMOD_H__ +#define __ACMOD_H__ + +/* System headers. */ +#include + +/* SphinxBase headers. */ +#include +#include +#include +#include +#include +#include +#include + +/* Local headers. */ +#include "ps_mllr.h" +#include "bin_mdef.h" +#include "tmat.h" +#include "hmm.h" + +/** + * States in utterance processing. + */ +typedef enum acmod_state_e { + ACMOD_IDLE, /**< Not in an utterance. */ + ACMOD_STARTED, /**< Utterance started, no data yet. */ + ACMOD_PROCESSING, /**< Utterance in progress. */ + ACMOD_ENDED /**< Utterance ended, still buffering. */ +} acmod_state_t; + +/** + * Dummy senone score value for unintentionally active states. + */ +#define SENSCR_DUMMY 0x7fff + +/** + * Feature space linear transform structure. + */ +struct ps_mllr_s { + int refcnt; /**< Reference count. */ + int n_class; /**< Number of MLLR classes. */ + int n_feat; /**< Number of feature streams. */ + int *veclen; /**< Length of input vectors for each stream. */ + float32 ****A; /**< Rotation part of mean transformations. */ + float32 ***b; /**< Bias part of mean transformations. */ + float32 ***h; /**< Diagonal transformation of variances. */ + int32 *cb2mllr; /**< Mapping from codebooks to transformations. */ +}; + +/** + * Acoustic model parameter structure. + */ +typedef struct ps_mgau_s ps_mgau_t; + +typedef struct ps_mgaufuncs_s { + char const *name; + + int (*frame_eval)(ps_mgau_t *mgau, + int16 *senscr, + uint8 *senone_active, + int32 n_senone_active, + mfcc_t ** feat, + int32 frame, + int32 compallsen); + int (*transform)(ps_mgau_t *mgau, + ps_mllr_t *mllr); + void (*free)(ps_mgau_t *mgau); +} ps_mgaufuncs_t; + +struct ps_mgau_s { + ps_mgaufuncs_t *vt; /**< vtable of mgau functions. */ + int frame_idx; /**< frame counter. */ +}; + +#define ps_mgau_base(mg) ((ps_mgau_t *)(mg)) +#define ps_mgau_frame_eval(mg,senscr,senone_active,n_senone_active,feat,frame,compallsen) \ + (*ps_mgau_base(mg)->vt->frame_eval) \ + (mg, senscr, senone_active, n_senone_active, feat, frame, compallsen) +#define ps_mgau_transform(mg, mllr) \ + (*ps_mgau_base(mg)->vt->transform)(mg, mllr) +#define ps_mgau_free(mg) \ + (*ps_mgau_base(mg)->vt->free)(mg) + +/** + * Acoustic model structure. + * + * This object encapsulates all stages of acoustic processing, from + * raw audio input to acoustic score output. The reason for grouping + * all of these modules together is that they all have to "agree" in + * their parameterizations, and the configuration of the acoustic and + * dynamic feature computation is completely dependent on the + * parameters used to build the original acoustic model (which should + * by now always be specified in a feat.params file). + * + * Because there is not a one-to-one correspondence from blocks of + * input audio or frames of input features to frames of acoustic + * scores (due to dynamic feature calculation), results may not be + * immediately available after input, and the output results will not + * correspond to the last piece of data input. + * + * TODO: In addition, this structure serves the purpose of queueing + * frames of features (and potentially also scores in the future) for + * asynchronous passes of recognition operating in parallel. + */ +struct acmod_s { + /* Global objects, not retained. */ + cmd_ln_t *config; /**< Configuration. */ + logmath_t *lmath; /**< Log-math computation. */ + glist_t strings; /**< Temporary acoustic model filenames. */ + + /* Feature computation: */ + fe_t *fe; /**< Acoustic feature computation. */ + feat_t *fcb; /**< Dynamic feature computation. */ + + /* Model parameters: */ + bin_mdef_t *mdef; /**< Model definition. */ + tmat_t *tmat; /**< Transition matrices. */ + ps_mgau_t *mgau; /**< Model parameters. */ + ps_mllr_t *mllr; /**< Speaker transformation. */ + + /* Senone scoring: */ + int16 *senone_scores; /**< GMM scores for current frame. */ + bitvec_t *senone_active_vec; /**< Active GMMs in current frame. */ + uint8 *senone_active; /**< Array of deltas to active GMMs. */ + int senscr_frame; /**< Frame index for senone_scores. */ + int n_senone_active; /**< Number of active GMMs. */ + int log_zero; /**< Zero log-probability value. */ + + /* Utterance processing: */ + mfcc_t **mfc_buf; /**< Temporary buffer of acoustic features. */ + mfcc_t ***feat_buf; /**< Temporary buffer of dynamic features. */ + FILE *rawfh; /**< File for writing raw audio data. */ + FILE *mfcfh; /**< File for writing acoustic feature data. */ + FILE *senfh; /**< File for writing senone score data. */ + FILE *insenfh; /**< Input senone score file. */ + long *framepos; /**< File positions of recent frames in senone file. */ + + /* Rawdata collected during decoding */ + int16 *rawdata; + int32 rawdata_size; + int32 rawdata_pos; + + /* A whole bunch of flags and counters: */ + uint8 state; /**< State of utterance processing. */ + uint8 compallsen; /**< Compute all senones? */ + uint8 grow_feat; /**< Whether to grow feat_buf. */ + uint8 insen_swap; /**< Whether to swap input senone score. */ + + frame_idx_t utt_start_frame; /**< Index of the utterance start in the stream, all timings are relative to that. */ + + frame_idx_t output_frame; /**< Index of next frame of dynamic features. */ + frame_idx_t n_mfc_alloc; /**< Number of frames allocated in mfc_buf */ + frame_idx_t n_mfc_frame; /**< Number of frames active in mfc_buf */ + frame_idx_t mfc_outidx; /**< Start of active frames in mfc_buf */ + frame_idx_t n_feat_alloc; /**< Number of frames allocated in feat_buf */ + frame_idx_t n_feat_frame; /**< Number of frames active in feat_buf */ + frame_idx_t feat_outidx; /**< Start of active frames in feat_buf */ +}; +typedef struct acmod_s acmod_t; + +/** + * Initialize an acoustic model. + * + * @param config a command-line object containing parameters. This + * pointer is not retained by this object. + * @param lmath global log-math parameters. + * @param fe a previously-initialized acoustic feature module to use, + * or NULL to create one automatically. If this is supplied + * and its parameters do not match those in the acoustic + * model, this function will fail. This pointer is not retained. + * @param fe a previously-initialized dynamic feature module to use, + * or NULL to create one automatically. If this is supplied + * and its parameters do not match those in the acoustic + * model, this function will fail. This pointer is not retained. + * @return a newly initialized acmod_t, or NULL on failure. + */ +acmod_t *acmod_init(cmd_ln_t *config, logmath_t *lmath, fe_t *fe, feat_t *fcb); + +/** + * Adapt acoustic model using a linear transform. + * + * @param mllr The new transform to use, or NULL to update the existing + * transform. The decoder retains ownership of this pointer, + * so you should not attempt to free it manually. Use + * ps_mllr_retain() if you wish to reuse it + * elsewhere. + * @return The updated transform object for this decoder, or + * NULL on failure. + */ +ps_mllr_t *acmod_update_mllr(acmod_t *acmod, ps_mllr_t *mllr); + +/** + * Start logging senone scores to a filehandle. + * + * @param acmod Acoustic model object. + * @param logfh Filehandle to log to. + * @return 0 for success, <0 on error. + */ +int acmod_set_senfh(acmod_t *acmod, FILE *senfh); + +/** + * Start logging MFCCs to a filehandle. + * + * @param acmod Acoustic model object. + * @param logfh Filehandle to log to. + * @return 0 for success, <0 on error. + */ +int acmod_set_mfcfh(acmod_t *acmod, FILE *logfh); + +/** + * Start logging raw audio to a filehandle. + * + * @param acmod Acoustic model object. + * @param logfh Filehandle to log to. + * @return 0 for success, <0 on error. + */ +int acmod_set_rawfh(acmod_t *acmod, FILE *logfh); + +/** + * Finalize an acoustic model. + */ +void acmod_free(acmod_t *acmod); + +/** + * Mark the start of an utterance. + */ +int acmod_start_utt(acmod_t *acmod); + +/** + * Mark the end of an utterance. + */ +int acmod_end_utt(acmod_t *acmod); + +/** + * Rewind the current utterance, allowing it to be rescored. + * + * After calling this function, the internal frame index is reset, and + * acmod_score() will return scores starting at the first frame of the + * current utterance. Currently, acmod_set_grow() must have been + * called to enable growing the feature buffer in order for this to + * work. In the future, senone scores may be cached instead. + * + * @return 0 for success, <0 for failure (if the utterance can't be + * rewound due to no feature or score data available) + */ +int acmod_rewind(acmod_t *acmod); + +/** + * Advance the frame index. + * + * This function moves to the next frame of input data. Subsequent + * calls to acmod_score() will return scores for that frame, until the + * next call to acmod_advance(). + * + * @return New frame index. + */ +int acmod_advance(acmod_t *acmod); + +/** + * Set memory allocation policy for utterance processing. + * + * @param grow_feat If non-zero, the internal dynamic feature buffer + * will expand as necessary to encompass any amount of data fed to the + * model. + * @return previous allocation policy. + */ +int acmod_set_grow(acmod_t *acmod, int grow_feat); + +/** + * TODO: Set queue length for utterance processing. + * + * This function allows multiple concurrent passes of search to + * operate on different parts of the utterance. + */ + +/** + * Feed raw audio data to the acoustic model for scoring. + * + * @param inout_raw In: Pointer to buffer of raw samples + * Out: Pointer to next sample to be read + * @param inout_n_samps In: Number of samples available + * Out: Number of samples remaining + * @param full_utt If non-zero, this block represents a full + * utterance and should be processed as such. + * @return Number of frames of data processed. + */ +int acmod_process_raw(acmod_t *acmod, + int16 const **inout_raw, + size_t *inout_n_samps, + int full_utt); + +/** + * Feed acoustic feature data into the acoustic model for scoring. + * + * @param inout_cep In: Pointer to buffer of features + * Out: Pointer to next frame to be read + * @param inout_n_frames In: Number of frames available + * Out: Number of frames remaining + * @param full_utt If non-zero, this block represents a full + * utterance and should be processed as such. + * @return Number of frames of data processed. + */ +int acmod_process_cep(acmod_t *acmod, + mfcc_t ***inout_cep, + int *inout_n_frames, + int full_utt); + +/** + * Feed dynamic feature data into the acoustic model for scoring. + * + * Unlike acmod_process_raw() and acmod_process_cep(), this function + * accepts a single frame at a time. This is because there is no need + * to do buffering when using dynamic features as input. However, if + * the dynamic feature buffer is full, this function will fail, so you + * should either always check the return value, or always pair a call + * to it with a call to acmod_score(). + * + * @param feat Pointer to one frame of dynamic features. + * @return Number of frames processed (either 0 or 1). + */ +int acmod_process_feat(acmod_t *acmod, + mfcc_t **feat); + +/** + * Set up a senone score dump file for input. + * + * @param insenfh File handle of dump file + * @return 0 for success, <0 for failure + */ +int acmod_set_insenfh(acmod_t *acmod, FILE *insenfh); + +/** + * Read one frame of scores from senone score dump file. + * + * @return Number of frames read or <0 on error. + */ +int acmod_read_scores(acmod_t *acmod); + +/** + * Get a frame of dynamic feature data. + * + * @param inout_frame_idx Input: frame index to get, or NULL + * to obtain features for the most recent frame. + * Output: frame index corresponding to this + * set of features. + * @return Feature array, or NULL if requested frame is not available. + */ +mfcc_t **acmod_get_frame(acmod_t *acmod, int *inout_frame_idx); + +/** + * Score one frame of data. + * + * @param inout_frame_idx Input: frame index to score, or NULL + * to obtain scores for the most recent frame. + * Output: frame index corresponding to this + * set of scores. + * @return Array of senone scores for this frame, or NULL if no frame + * is available for scoring (such as if a frame index is + * requested that is not yet or no longer available). The + * data pointed to persists only until the next call to + * acmod_score() or acmod_advance(). + */ +int16 const *acmod_score(acmod_t *acmod, + int *inout_frame_idx); + +/** + * Write senone dump file header. + */ +int acmod_write_senfh_header(acmod_t *acmod, FILE *logfh); + +/** + * Write a frame of senone scores to a dump file. + */ +int acmod_write_scores(acmod_t *acmod, int n_active, uint8 const *active, + int16 const *senscr, FILE *senfh); + + +/** + * Get best score and senone index for current frame. + */ +int acmod_best_score(acmod_t *acmod, int *out_best_senid); + +/** + * Clear set of active senones. + */ +void acmod_clear_active(acmod_t *acmod); + +/** + * Activate senones associated with an HMM. + */ +void acmod_activate_hmm(acmod_t *acmod, hmm_t *hmm); + +/** + * Activate a single senone. + */ +#define acmod_activate_sen(acmod, sen) bitvec_set((acmod)->senone_active_vec, sen) + +/** + * Build active list from + */ +int32 acmod_flags2list(acmod_t *acmod); + +/** + * Get the offset of the utterance start of the current stream, helpful for stream-wide timing. + */ +int32 acmod_stream_offset(acmod_t *acmod); + +/** + * Reset the current stream + */ +void acmod_start_stream(acmod_t *acmod); + +/** + * Sets the limit of the raw audio data to store + */ +void acmod_set_rawdata_size(acmod_t *acmod, int32 size); + +/** + * Retrieves the raw data collected during utterance decoding + */ +void acmod_get_rawdata(acmod_t *acmod, int16 **buffer, int32 *size); + +#endif /* __ACMOD_H__ */ diff --git a/include/allphone_search.h b/include/allphone_search.h new file mode 100644 index 0000000..d09a4e3 --- /dev/null +++ b/include/allphone_search.h @@ -0,0 +1,179 @@ +/* -*- c-basic-offset:4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 2014 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +/* + * allphone_search.h -- Search structures for phoneme decoding. + */ + + +#ifndef __ALLPHONE_SEARCH_H__ +#define __ALLPHONE_SEARCH_H__ + + +/* SphinxBase headers. */ +#include +#include +#include +#include + +/* Local headers. */ +#include "pocketsphinx_internal.h" +#include "blkarray_list.h" +#include "hmm.h" + +/** + * Models a single unique pair. + * Can represent several different triphones, but all with the same parent basephone. + * (NOTE: Word-position attribute of triphone is ignored.) + */ +typedef struct phmm_s { + hmm_t hmm; /**< Base HMM structure */ + s3pid_t pid; /**< Phone id (temp. during init.) */ + s3cipid_t ci; /**< Parent basephone for this PHMM */ + bitvec_t *lc; /**< Set (bit-vector) of left context phones seen for this PHMM */ + bitvec_t *rc; /**< Set (bit-vector) of right context phones seen for this PHMM */ + struct phmm_s *next; /**< Next unique PHMM for same parent basephone */ + struct plink_s *succlist; /**< List of predecessor PHMM nodes */ +} phmm_t; + +/** + * List of links from a PHMM node to its successors; one link per successor. + */ +typedef struct plink_s { + phmm_t *phmm; /**< Successor PHMM node */ + struct plink_s *next; /**< Next link for parent PHMM node */ +} plink_t; + +/** + * History (paths) information at any point in allphone Viterbi search. + */ +typedef struct history_s { + phmm_t *phmm; /**< PHMM ending this path */ + int32 score; /**< Path score for this path */ + int32 tscore; /**< Transition score for this path */ + frame_idx_t ef; /**< End frame */ + int32 hist; /**< Previous history entry */ +} history_t; + +/** + * Phone level segmentation information + */ +typedef struct phseg_s { + s3cipid_t ci; /* CI-phone id */ + frame_idx_t sf, ef; /* Start and end frame for this phone occurrence */ + int32 score; /* Acoustic score for this segment of alignment */ + int32 tscore; /* Transition ("LM") score for this segment */ +} phseg_t; + +/** + * Segment iterator over list of phseg + */ +typedef struct phseg_iter_s { + ps_seg_t base; + glist_t seg; +} phseg_iter_t; + +/** + * Implementation of allphone search structure. + */ +typedef struct allphone_search_s { + ps_search_t base; + + hmm_context_t *hmmctx; /**< HMM context. */ + ngram_model_t *lm; /**< Ngram model set */ + int32 ci_only; /**< Use context-independent phones for decoding */ + phmm_t **ci_phmm; /**< PHMM lists (for each CI phone) */ + int32 *ci2lmwid; /**< Mapping of CI phones to LM word IDs */ + + int32 beam, pbeam; /**< Effective beams after applying beam_factor */ + int32 lw, inspen; /**< Language weights */ + + frame_idx_t frame; /**< Current frame. */ + float32 ascale; /**< Acoustic score scale for posterior probabilities. */ + + int32 n_tot_frame; /**< Total number of frames processed */ + int32 n_hmm_eval; /**< Total HMMs evaluated this utt */ + int32 n_sen_eval; /**< Total senones evaluated this utt */ + + /* Backtrace information */ + blkarray_list_t *history; /**< List of history nodes allocated in each frame */ + /* Hypothesis DAG */ + glist_t segments; + + ptmr_t perf; /**< Performance counter */ + +} allphone_search_t; + +/** + * Create, initialize and return a search module. + */ +ps_search_t *allphone_search_init(const char *name, + ngram_model_t * lm, + cmd_ln_t * config, + acmod_t * acmod, + dict_t * dict, dict2pid_t * d2p); + +/** + * Deallocate search structure. + */ +void allphone_search_free(ps_search_t * search); + +/** + * Update allphone search module. + */ +int allphone_search_reinit(ps_search_t * search, dict_t * dict, + dict2pid_t * d2p); + +/** + * Prepare the allphone search structure for beginning decoding of the next + * utterance. + */ +int allphone_search_start(ps_search_t * search); + +/** + * Step one frame forward through the Viterbi search. + */ +int allphone_search_step(ps_search_t * search, int frame_idx); + +/** + * Windup and clean the allphone search structure after utterance. + */ +int allphone_search_finish(ps_search_t * search); + +/** + * Get hypothesis string from the allphone search. + */ +char const *allphone_search_hyp(ps_search_t * search, int32 * out_score); + +#endif /* __ALLPHONE_SEARCH_H__ */ diff --git a/include/bin_mdef.h b/include/bin_mdef.h new file mode 100644 index 0000000..a22aa2e --- /dev/null +++ b/include/bin_mdef.h @@ -0,0 +1,236 @@ +/* -*- c-file-style: "linux" -*- */ +/* ==================================================================== + * Copyright (c) 2005 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ +/** + * @file bin_mdef.h + * + * Binary format model definition files, with support for + * heterogeneous topologies and variable-size N-phones + * + * @author David Huggins-Daines + */ +#ifndef __BIN_MDEF_H__ +#define __BIN_MDEF_H__ + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +/* SphinxBase headers. */ +#include +#include +#include + +#include "mdef.h" + +#define BIN_MDEF_FORMAT_VERSION 1 +/* Little-endian machines will write "BMDF" to disk, big-endian ones "FDMB". */ +#define BIN_MDEF_NATIVE_ENDIAN 0x46444d42 /* 'BMDF' in little-endian order */ +#define BIN_MDEF_OTHER_ENDIAN 0x424d4446 /* 'BMDF' in big-endian order */ +#ifdef __GNUC__ +#define __ATTRIBUTE_PACKED __attribute__((packed)) +#else +#define __ATTRIBUTE_PACKED +#endif + +/** + * Phone entry (on-disk, 12 bytes) + */ +typedef struct mdef_entry_s mdef_entry_t; +struct mdef_entry_s { + int32 ssid; /**< Senone sequence ID */ + int32 tmat; /**< Transition matrix ID */ + /* FIXME: is any of this actually necessary? */ + union { + /**< CI phone information - attributes (just "filler" for now) */ + struct { + uint8 filler; + uint8 reserved[3]; + } ci; + /**< CD phone information - context info. */ + struct { + uint8 wpos; + uint8 ctx[3]; /**< quintphones will require hacking */ + } cd; + } info; +} __ATTRIBUTE_PACKED; + +/** + * Invalid senone sequence ID (limited to 16 bits for PocketSphinx). + */ +#define BAD_SSID 0xffff +/** + * Invalid senone ID (limited to 16 bits for PocketSphinx). + */ +#define BAD_SENID 0xffff + +/** + * Node in CD phone tree (on-disk, 8 bytes). + */ +typedef struct cd_tree_s cd_tree_t; +struct cd_tree_s { + int16 ctx; /**< Context (word position or CI phone) */ + int16 n_down; /**< Number of children (0 for leafnode) */ + union { + int32 pid; /**< Phone ID (leafnode) */ + int32 down; /**< Next level of the tree (offset from start of cd_trees) */ + } c; +}; + +/** + * Model definition structure (in-memory). + */ +typedef struct bin_mdef_s bin_mdef_t; +struct bin_mdef_s { + int refcnt; + int32 n_ciphone; /**< Number of base (CI) phones */ + int32 n_phone; /**< Number of base (CI) phones + (CD) triphones */ + int32 n_emit_state; /**< Number of emitting states per phone (0 for heterogeneous) */ + int32 n_ci_sen; /**< Number of CI senones; these are the first */ + int32 n_sen; /**< Number of senones (CI+CD) */ + int32 n_tmat; /**< Number of transition matrices */ + int32 n_sseq; /**< Number of unique senone sequences */ + int32 n_ctx; /**< Number of phones of context */ + int32 n_cd_tree; /**< Number of nodes in cd_tree (below) */ + int32 sil; /**< CI phone ID for silence */ + + mmio_file_t *filemap;/**< File map for this file (if any) */ + char **ciname; /**< CI phone names */ + cd_tree_t *cd_tree; /**< Tree mapping CD phones to phone IDs */ + mdef_entry_t *phone; /**< All phone structures */ + uint16 **sseq; /**< Unique senone sequences (2D array built at load time) */ + uint8 *sseq_len; /**< Number of states in each sseq (NULL for homogeneous) */ + + /* These two are not stored on disk, but are generated at load time. */ + int16 *cd2cisen; /**< Parent CI-senone id for each senone */ + int16 *sen2cimap; /**< Parent CI-phone for each senone (CI or CD) */ + + /** Allocation mode for this object. */ + enum { BIN_MDEF_FROM_TEXT, BIN_MDEF_IN_MEMORY, BIN_MDEF_ON_DISK } alloc_mode; +}; + +#define bin_mdef_is_fillerphone(m,p) (((p) < (m)->n_ciphone) \ + ? (m)->phone[p].info.ci.filler \ + : (m)->phone[(m)->phone[p].info.cd.ctx[0]].info.ci.filler) +#define bin_mdef_is_ciphone(m,p) ((p) < (m)->n_ciphone) +#define bin_mdef_n_ciphone(m) ((m)->n_ciphone) +#define bin_mdef_n_phone(m) ((m)->n_phone) +#define bin_mdef_n_sseq(m) ((m)->n_sseq) +#define bin_mdef_n_emit_state(m) ((m)->n_emit_state) +#define bin_mdef_n_emit_state_phone(m,p) ((m)->n_emit_state ? (m)->n_emit_state \ + : (m)->sseq_len[(m)->phone[p].ssid]) +#define bin_mdef_n_sen(m) ((m)->n_sen) +#define bin_mdef_n_tmat(m) ((m)->n_tmat) +#define bin_mdef_pid2ssid(m,p) ((m)->phone[p].ssid) +#define bin_mdef_pid2tmatid(m,p) ((m)->phone[p].tmat) +#define bin_mdef_silphone(m) ((m)->sil) +#define bin_mdef_sen2cimap(m,s) ((m)->sen2cimap[s]) +#define bin_mdef_sseq2sen(m,ss,pos) ((m)->sseq[ss][pos]) +#define bin_mdef_pid2ci(m,p) (((p) < (m)->n_ciphone) ? (p) \ + : (m)->phone[p].info.cd.ctx[0]) + +/** + * Read a binary mdef from a file. + */ +POCKETSPHINX_EXPORT +bin_mdef_t *bin_mdef_read(cmd_ln_t *config, const char *filename); +/** + * Read a text mdef from a file (creating an in-memory binary mdef). + */ +POCKETSPHINX_EXPORT +bin_mdef_t *bin_mdef_read_text(cmd_ln_t *config, const char *filename); +/** + * Write a binary mdef to a file. + */ +POCKETSPHINX_EXPORT +int bin_mdef_write(bin_mdef_t *m, const char *filename); +/** + * Write a binary mdef to a text file. + */ +POCKETSPHINX_EXPORT +int bin_mdef_write_text(bin_mdef_t *m, const char *filename); +/** + * Retain a pointer to a bin_mdef_t. + */ +bin_mdef_t *bin_mdef_retain(bin_mdef_t *m); +/** + * Release a pointer to a binary mdef. + */ +int bin_mdef_free(bin_mdef_t *m); + +/** + * Context-independent phone lookup. + * @return phone id for ciphone. + */ +int bin_mdef_ciphone_id(bin_mdef_t *m, /**< In: Model structure being queried */ + const char *ciphone); /**< In: ciphone for which id wanted */ + +/** + * Case-insensitive context-independent phone lookup. + * @return phone id for ciphone. + */ +int bin_mdef_ciphone_id_nocase(bin_mdef_t *m, /**< In: Model structure being queried */ + const char *ciphone); /**< In: ciphone for which id wanted */ + +/* Return value: READ-ONLY ciphone string name for the given ciphone id */ +const char *bin_mdef_ciphone_str(bin_mdef_t *m, /**< In: Model structure being queried */ + int32 ci); /**< In: ciphone id for which name wanted */ + +/* Return value: phone id for the given constituents if found, else -1 */ +int bin_mdef_phone_id(bin_mdef_t *m, /**< In: Model structure being queried */ + int32 b, /**< In: base ciphone id */ + int32 l, /**< In: left context ciphone id */ + int32 r, /**< In: right context ciphone id */ + int32 pos); /**< In: Word position */ + +/* Look up a phone id, backing off to other word positions. */ +int bin_mdef_phone_id_nearest(bin_mdef_t * m, int32 b, + int32 l, int32 r, int32 pos); + +/** + * Create a phone string for the given phone (base or triphone) id in the given buf. + * + * @return 0 if successful, -1 if error. + */ +int bin_mdef_phone_str(bin_mdef_t *m, /**< In: Model structure being queried */ + int pid, /**< In: phone id being queried */ + char *buf); /**< Out: On return, buf has the string */ + +#ifdef __cplusplus +}; /* extern "C" */ +#endif /* __cplusplus */ + +#endif /* __BIN_MDEF_H__ */ diff --git a/include/blkarray_list.h b/include/blkarray_list.h new file mode 100644 index 0000000..a2e8513 --- /dev/null +++ b/include/blkarray_list.h @@ -0,0 +1,139 @@ +/* ==================================================================== + * Copyright (c) 1999-2004 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +/* + * blkarray_list.h -- array-based list structure, for memory and access + * efficiency. + * + * HISTORY + * + * $Log: blkarray_list.h,v $ + * Revision 1.1.1.1 2006/05/23 18:45:02 dhuggins + * re-importation + * + * Revision 1.2 2004/12/10 16:48:58 rkm + * Added continuous density acoustic model handling + * + * Revision 1.1 2004/07/16 00:57:12 egouvea + * Added Ravi's implementation of FSG support. + * + * Revision 1.2 2004/05/27 14:22:57 rkm + * FSG cross-word triphones completed (but for single-phone words) + * + * Revision 1.1.1.1 2004/03/01 14:30:31 rkm + * + * + * Revision 1.1 2004/02/26 01:14:48 rkm + * *** empty log message *** + * + * + * 18-Feb-2004 M K Ravishankar (rkm@cs.cmu.edu) at Carnegie Mellon + * Started. + */ + + +#ifndef __S2_BLKARRAY_LIST_H__ +#define __S2_BLKARRAY_LIST_H__ + + +#include + + +/* + * For maintaining a (conceptual) "list" of pointers to arbitrary data. + * The application is responsible for knowing the true data type. + * Use an array instead of a true list for efficiency (both memory and + * speed). But use a blocked (2-D) array to allow dynamic resizing at a + * coarse grain. An entire block is allocated or freed, as appropriate. + */ +typedef struct blkarray_list_s { + void ***ptr; /* ptr[][] is the user-supplied ptr */ + int32 maxblks; /* size of ptr (#rows) */ + int32 blksize; /* size of ptr[] (#cols, ie, size of each row) */ + int32 n_valid; /* # entries actually stored in the list */ + int32 cur_row; /* The current row being that has empty entry */ + int32 cur_row_free; /* First entry valid within the current row */ +} blkarray_list_t; + +/* Access macros */ +#define blkarray_list_ptr(l,r,c) ((l)->ptr[r][c]) +#define blkarray_list_maxblks(l) ((l)->maxblks) +#define blkarray_list_blksize(l) ((l)->blksize) +#define blkarray_list_n_valid(l) ((l)->n_valid) +#define blkarray_list_cur_row(l) ((l)->cur_row) +#define blkarray_list_cur_row_free(l) ((l)->cur_row_free) + + +/* + * Initialize and return a new blkarray_list containing an empty list + * (i.e., 0 length). Sized for the given values of maxblks and blksize. + * NOTE: (maxblks * blksize) should not overflow int32, but this is not + * checked. + * Return the allocated entry if successful, NULL if any error. + */ +blkarray_list_t *_blkarray_list_init (int32 maxblks, int32 blksize); + + +/* + * Like _blkarray_list_init() above, but for some default values of + * maxblks and blksize. + */ +blkarray_list_t *blkarray_list_init ( void ); + +/** + * Completely finalize a blkarray_list. + */ +void blkarray_list_free(blkarray_list_t *bl); + + +/* + * Append the given new entry (data) to the end of the list. + * Return the index of the entry if successful, -1 if any error. + * The returned indices are guaranteed to be successive integers (i.e., + * 0, 1, 2...) for successive append operations, until the list is reset, + * when they resume from 0. + */ +int32 blkarray_list_append (blkarray_list_t *, void *data); + + +/* + * Free all the entries in the list (using ckd_free) and reset the + * list length to 0. + */ +void blkarray_list_reset (blkarray_list_t *); + + +/* Gets n-th element of the array list */ +void * blkarray_list_get(blkarray_list_t *, int32 n); + +#endif diff --git a/include/dict.h b/include/dict.h new file mode 100644 index 0000000..26ffd2b --- /dev/null +++ b/include/dict.h @@ -0,0 +1,210 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 1999-2004 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +#ifndef _S3_DICT_H_ +#define _S3_DICT_H_ + +/** \file dict.h + * \brief Operations on dictionary. + */ + +/* SphinxBase headers. */ +#include + +/* Local headers. */ +#include "s3types.h" +#include "bin_mdef.h" +#include "pocketsphinx_export.h" + +#define S3DICT_INC_SZ 4096 + +#ifdef __cplusplus +extern "C" { +#endif + +/** + \struct dictword_t + \brief a structure for one dictionary word. +*/ +typedef struct { + char *word; /**< Ascii word string */ + s3cipid_t *ciphone; /**< Pronunciation */ + int32 pronlen; /**< Pronunciation length */ + s3wid_t alt; /**< Next alternative pronunciation id, NOT_S3WID if none */ + s3wid_t basewid; /**< Base pronunciation id */ +} dictword_t; + +/** + \struct dict_t + \brief a structure for a dictionary. +*/ + +typedef struct { + int refcnt; + bin_mdef_t *mdef; /**< Model definition used for phone IDs; NULL if none used */ + dictword_t *word; /**< Array of entries in dictionary */ + hash_table_t *ht; /**< Hash table for mapping word strings to word ids */ + int32 max_words; /**< #Entries allocated in dict, including empty slots */ + int32 n_word; /**< #Occupied entries in dict; ie, excluding empty slots */ + int32 filler_start; /**< First filler word id (read from filler dict) */ + int32 filler_end; /**< Last filler word id (read from filler dict) */ + s3wid_t startwid; /**< FOR INTERNAL-USE ONLY */ + s3wid_t finishwid; /**< FOR INTERNAL-USE ONLY */ + s3wid_t silwid; /**< FOR INTERNAL-USE ONLY */ + int nocase; +} dict_t; + + +/** + * Initialize a new dictionary. + * + * If config and mdef are supplied, then the dictionary will be read + * from the files specified by the -dict and -fdict options in config, + * with case sensitivity determined by the -dictcase option. + * + * Otherwise an empty case-sensitive dictionary will be created. + * + * Return ptr to dict_t if successful, NULL otherwise. + */ +dict_t *dict_init(cmd_ln_t *config, /**< Configuration (-dict, -fdict, -dictcase) or NULL */ + bin_mdef_t *mdef /**< For looking up CI phone IDs (or NULL) */ + ); + +/** + * Write dictionary to a file. + */ +int dict_write(dict_t *dict, char const *filename, char const *format); + +/** Return word id for given word string if present. Otherwise return BAD_S3WID */ +POCKETSPHINX_EXPORT +s3wid_t dict_wordid(dict_t *d, const char *word); + +/** + * Return 1 if w is a filler word, 0 if not. A filler word is one that was read in from the + * filler dictionary; however, sentence START and FINISH words are not filler words. + */ +int dict_filler_word(dict_t *d, /**< The dictionary structure */ + s3wid_t w /**< The word ID */ + ); + +/** + * Test if w is a "real" word, i.e. neither a filler word nor START/FINISH. + */ +POCKETSPHINX_EXPORT +int dict_real_word(dict_t *d, /**< The dictionary structure */ + s3wid_t w /**< The word ID */ + ); + +/** + * Add a word with the given ciphone pronunciation list to the dictionary. + * Return value: Result word id if successful, BAD_S3WID otherwise + */ +s3wid_t dict_add_word(dict_t *d, /**< The dictionary structure. */ + char const *word, /**< The word. */ + s3cipid_t const *p, /**< The pronunciation. */ + int32 np /**< Number of phones. */ + ); + +/** + * Return value: CI phone string for the given word, phone position. + */ +const char *dict_ciphone_str(dict_t *d, /**< In: Dictionary to look up */ + s3wid_t wid, /**< In: Component word being looked up */ + int32 pos /**< In: Pronunciation phone position */ + ); + +/** Packaged macro access to dictionary members */ +#define dict_size(d) ((d)->n_word) +#define dict_num_fillers(d) (dict_filler_end(d) - dict_filler_start(d)) +/** + * Number of "real words" in the dictionary. + * + * This is the number of words that are not fillers, , or . + */ +#define dict_num_real_words(d) \ + (dict_size(d) - (dict_filler_end(d) - dict_filler_start(d)) - 2) +#define dict_basewid(d,w) ((d)->word[w].basewid) +#define dict_wordstr(d,w) ((w) < 0 ? NULL : (d)->word[w].word) +#define dict_basestr(d,w) ((d)->word[dict_basewid(d,w)].word) +#define dict_nextalt(d,w) ((d)->word[w].alt) +#define dict_pronlen(d,w) ((d)->word[w].pronlen) +#define dict_pron(d,w,p) ((d)->word[w].ciphone[p]) /**< The CI phones of the word w at position p */ +#define dict_filler_start(d) ((d)->filler_start) +#define dict_filler_end(d) ((d)->filler_end) +#define dict_startwid(d) ((d)->startwid) +#define dict_finishwid(d) ((d)->finishwid) +#define dict_silwid(d) ((d)->silwid) +#define dict_is_single_phone(d,w) ((d)->word[w].pronlen == 1) +#define dict_first_phone(d,w) ((d)->word[w].ciphone[0]) +#define dict_second_phone(d,w) ((d)->word[w].ciphone[1]) +#define dict_second_last_phone(d,w) ((d)->word[w].ciphone[(d)->word[w].pronlen - 2]) +#define dict_last_phone(d,w) ((d)->word[w].ciphone[(d)->word[w].pronlen - 1]) + +/* Hard-coded special words */ +#define S3_START_WORD "" +#define S3_FINISH_WORD "" +#define S3_SILENCE_WORD "" +#define S3_UNKNOWN_WORD "" + +/** + * If the given word contains a trailing "(....)" (i.e., a Sphinx-II style alternative + * pronunciation specification), strip that trailing portion from it. Note that the given + * string is modified. + * Return value: If string was modified, the character position at which the original string + * was truncated; otherwise -1. + */ +int32 dict_word2basestr(char *word); + +/** + * Retain a pointer to an dict_t. + */ +dict_t *dict_retain(dict_t *d); + +/** + * Release a pointer to a dictionary. + */ +int dict_free(dict_t *d); + +/** Report a dictionary structure */ +void dict_report(dict_t *d /**< A dictionary structure */ + ); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/dict2pid.h b/include/dict2pid.h new file mode 100644 index 0000000..f81cf63 --- /dev/null +++ b/include/dict2pid.h @@ -0,0 +1,180 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 1999-2014 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +#ifndef _S3_DICT2PID_H_ +#define _S3_DICT2PID_H_ + +/* System headers. */ +#include + +/* SphinxBase headers. */ +#include +#include + +/* Local headers. */ +#include "s3types.h" +#include "bin_mdef.h" +#include "dict.h" + +/** \file dict2pid.h + * \brief Building triphones for a dictionary. + * + * This is one of the more complicated parts of a cross-word + * triphone model decoder. The first and last phones of each word + * get their left and right contexts, respectively, from other + * words. For single-phone words, both its contexts are from other + * words, simultaneously. As these words are not known beforehand, + * life gets complicated. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * \struct xwdssid_t + * \brief cross word triphone model structure + */ + +typedef struct { + s3ssid_t *ssid; /**< Senone Sequence ID list for all context ciphones */ + s3cipid_t *cimap; /**< Index into ssid[] above for each ci phone */ + int32 n_ssid; /**< #Unique ssid in above, compressed ssid list */ +} xwdssid_t; + +/** + \struct dict2pid_t + \brief Building composite triphone (as well as word internal triphones) with the dictionary. +*/ + +typedef struct { + int refcount; + + bin_mdef_t *mdef; /**< Model definition, used to generate + internal ssids on the fly. */ + dict_t *dict; /**< Dictionary this table refers to. */ + + /*Notice the order of the arguments */ + /* FIXME: This is crying out for compression - in Mandarin we have + * 180 context independent phones, which makes this an 11MB + * array. */ + s3ssid_t ***ldiph_lc; /**< For multi-phone words, [base][rc][lc] -> ssid; filled out for + word-initial base x rc combinations in current vocabulary */ + + + xwdssid_t **rssid; /**< Right context state sequence id table + First dimension: base phone, + Second dimension: left context. + */ + + + s3ssid_t ***lrdiph_rc; /**< For single-phone words, [base][lc][rc] -> ssid; filled out for + single-phone base x lc combinations in current vocabulary */ + + xwdssid_t **lrssid; /**< Left-Right context state sequence id table + First dimension: base phone, + Second dimension: left context. + */ +} dict2pid_t; + +/** Access macros; not designed for arbitrary use */ +#define dict2pid_rssid(d,ci,lc) (&(d)->rssid[ci][lc]) +#define dict2pid_ldiph_lc(d,b,r,l) ((d)->ldiph_lc[b][r][l]) +#define dict2pid_lrdiph_rc(d,b,l,r) ((d)->lrdiph_rc[b][l][r]) + +/** + * Build the dict2pid structure for the given model/dictionary + */ +dict2pid_t *dict2pid_build(bin_mdef_t *mdef, /**< A model definition*/ + dict_t *dict /**< An initialized dictionary */ + ); + +/** + * Retain a pointer to dict2pid + */ +dict2pid_t *dict2pid_retain(dict2pid_t *d2p); + +/** + * Free the memory dict2pid structure + */ +int dict2pid_free(dict2pid_t *d2p /**< In: the d2p */ + ); + +/** + * Return the senone sequence ID for the given word position. + */ +s3ssid_t dict2pid_internal(dict2pid_t *d2p, + int32 wid, + int pos); + +/** + * Add a word to the dict2pid structure (after adding it to dict). + */ +int dict2pid_add_word(dict2pid_t *d2p, + int32 wid); + +/** + * For debugging + */ +void dict2pid_dump(FILE *fp, /**< In: a file pointer */ + dict2pid_t *d2p /**< In: a dict2pid_t structure */ + ); + +/** Report a dict2pid data structure */ +void dict2pid_report(dict2pid_t *d2p /**< In: a dict2pid_t structure */ + ); + +/** + * Get number of rc + */ +int32 get_rc_nssid(dict2pid_t *d2p, /**< In: a dict2pid */ + s3wid_t w /**< In: a wid */ + ); + +/** + * Get RC map + */ +s3cipid_t* dict2pid_get_rcmap(dict2pid_t *d2p, /**< In: a dict2pid */ + s3wid_t w /**< In: a wid */ + ); + +#ifdef __cplusplus +} +#endif + + +#endif diff --git a/include/fsg_history.h b/include/fsg_history.h new file mode 100644 index 0000000..5eaad65 --- /dev/null +++ b/include/fsg_history.h @@ -0,0 +1,215 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 1999-2004 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ +/* + * fsg_history.h -- FSG Viterbi decode history + * + * ********************************************** + * CMU ARPA Speech Project + * + * Copyright (c) 1999 Carnegie Mellon University. + * ALL RIGHTS RESERVED. + * ********************************************** + * + * HISTORY + * + * $Log: fsg_history.h,v $ + * Revision 1.1.1.1 2006/05/23 18:45:02 dhuggins + * re-importation + * + * Revision 1.1 2004/07/16 00:57:12 egouvea + * Added Ravi's implementation of FSG support. + * + * Revision 1.7 2004/07/07 22:30:35 rkm + * *** empty log message *** + * + * Revision 1.6 2004/07/07 13:56:33 rkm + * Added reporting of (acoustic score - best senone score)/frame + * + * Revision 1.5 2004/06/25 14:49:08 rkm + * Optimized size of history table and speed of word transitions by maintaining only best scoring word exits at each state + * + * Revision 1.4 2004/06/23 20:32:16 rkm + * *** empty log message *** + * + * Revision 1.3 2004/05/27 15:16:08 rkm + * *** empty log message *** + * + * + * 25-Feb-2004 M K Ravishankar (rkm@cs.cmu.edu) at Carnegie Mellon University + * Started, based on S3.3 version. + */ + + +#ifndef __S2_FSG_HISTORY_H__ +#define __S2_FSG_HISTORY_H__ + + +/* SphinxBase headers. */ +#include +#include + +/* Local headers. */ +#include "blkarray_list.h" +#include "fsg_lextree.h" +#include "dict.h" + +/* + * The Viterbi history structure. This is a tree, with the root at the + * FSG start state, at frame 0, with a null predecessor. + */ + +/* + * A single Viterbi history entry + */ +typedef struct fsg_hist_entry_s { + fsg_link_t *fsglink; /* Link taken result in this entry */ + int32 score; /* Total path score at the end of this + transition */ + int32 pred; /* Predecessor entry; -1 if none */ + frame_idx_t frame; /* Ending frame for this entry */ + int16 lc; /* Left context provided by this entry to + succeeding words */ + fsg_pnode_ctxt_t rc; /* Possible right contexts to which this entry + applies */ +} fsg_hist_entry_t; + +/* Access macros */ +#define fsg_hist_entry_fsglink(v) ((v)->fsglink) +#define fsg_hist_entry_frame(v) ((v)->frame) +#define fsg_hist_entry_score(v) ((v)->score) +#define fsg_hist_entry_pred(v) ((v)->pred) +#define fsg_hist_entry_lc(v) ((v)->lc) +#define fsg_hist_entry_rc(v) ((v)->rc) + + +/* + * The entire tree of history entries (fsg_history_t.entries). + * Optimization: In a given frame, there may be several history entries, with + * the same left and right phonetic context, terminating in a particular state. + * Only the best scoring one of these needs to be saved, since everything else + * will be pruned according to the Viterbi algorithm. frame_entries is used + * temporarily in each frame to determine these best scoring entries in that + * frame. Only the ones not pruned are transferred to entries at the end of + * the frame. However, null transitions are a problem since they create + * entries that depend on entries created in the CURRENT frame. Hence, this + * pruning is done in two stages: first for the non-null transitions, and then + * for the null transitions alone. (This solution is sub-optimal, and can be + * improved with a little more work. SMOP.) + * Why is frame_entries a list? Each entry has a unique terminating state, + * and has a unique lc CIphone. But it has a SET of rc CIphones. + * frame_entries[s][lc] is an ordered list of entries created in the current + * frame, terminating in state s, and with left context lc. The list is in + * descending order of path score. When a new entry with (s,lc) arrives, + * its position in the list is determined. Then its rc set is modified by + * subtracting the union of the rc's of all its predecessors (i.e., better + * scoring entries). If the resulting rc set is empty, the entry is discarded. + * Otherwise, it is inserted, and the rc sets of all downstream entries in the + * list are updated by subtracting the new entry's rc. If any of them becomes + * empty, it is also discarded. + * As mentioned earlier, this procedure is applied in two stages, for the + * non-null transitions, and the null transitions, separately. + */ +typedef struct fsg_history_s { + fsg_model_t *fsg; /* The FSG for which this object applies */ + blkarray_list_t *entries; /* A list of history table entries; the root + entry is the first element of the list */ + glist_t **frame_entries; + int n_ciphone; +} fsg_history_t; + + +/* + * One-time intialization: Allocate and return an initially empty history + * module. + */ +fsg_history_t *fsg_history_init(fsg_model_t *fsg, dict_t *dict); + +void fsg_history_utt_start(fsg_history_t *h); + +void fsg_history_utt_end(fsg_history_t *h); + + +/* + * Create a history entry recording the completion of the given FSG + * transition, at the end of the given frame, with the given score, and + * the given predecessor history entry. + * The entry is initially temporary, and may be superseded by another + * with a higher score. The surviving entries must be transferred to + * the main history table, via fsg_history_end_frame(). + */ +void fsg_history_entry_add (fsg_history_t *h, + fsg_link_t *l, /* FSG transition */ + int32 frame, + int32 score, + int32 pred, + int32 lc, + fsg_pnode_ctxt_t rc); + +/* + * Transfer the surviving history entries for this frame into the permanent + * history table. This function can be called several times during a frame. + * Each time, the entries surviving so far are transferred, and the temporary + * lists cleared. This feature is used to handle the entries due to non-null + * transitions and null transitions separately. + */ +void fsg_history_end_frame (fsg_history_t *h); + + +/* Clear the hitory table */ +void fsg_history_reset (fsg_history_t *h); + + +/* Return the number of valid entries in the given history table */ +int32 fsg_history_n_entries (fsg_history_t *h); + +/* + * Return a ptr to the history entry for the given ID; NULL if there is no + * such entry. + */ +fsg_hist_entry_t *fsg_history_entry_get(fsg_history_t *h, int32 id); + + +/* + * Switch the FSG associated with the given history module. Should be done + * when the history list is empty. If not empty, the list is cleared. + */ +void fsg_history_set_fsg (fsg_history_t *h, fsg_model_t *fsg, dict_t *dict); + +/* Free the given Viterbi search history object */ +void fsg_history_free (fsg_history_t *h); + +/* Print the entire history */ +void fsg_history_print(fsg_history_t *h, dict_t *dict); + +#endif diff --git a/include/fsg_lextree.h b/include/fsg_lextree.h new file mode 100644 index 0000000..563065c --- /dev/null +++ b/include/fsg_lextree.h @@ -0,0 +1,255 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 1999-2013 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ +/* + * fsg_lextree.h -- The collection of all the lextrees for the entire FSM. + * + */ + +#ifndef __S2_FSG_LEXTREE_H__ +#define __S2_FSG_LEXTREE_H__ + +/* SphinxBase headers. */ +#include +#include + +/* Local headers. */ +#include "hmm.h" +#include "dict.h" +#include "dict2pid.h" + +/* + * Compile-time constant determining the size of the + * bitvector fsg_pnode_t.fsg_pnode_ctxt_t.bv. (See below.) + * But it makes memory allocation simpler and more efficient. + * Make it smaller (2) to save memory if your phoneset has less than + * 64 phones. + */ +#define FSG_PNODE_CTXT_BVSZ 4 + +typedef struct { + uint32 bv[FSG_PNODE_CTXT_BVSZ]; +} fsg_pnode_ctxt_t; + + +/* + * All transitions (words) out of any given FSG state represented are by a + * phonetic prefix lextree (except for epsilon or null transitions; they + * are not part of the lextree). Lextree leaf nodes represent individual + * FSG transitions, so no sharing is allowed at the leaf nodes. The FSG + * transition probs are distributed along the lextree: the prob at a node + * is the max of the probs of all leaf nodes (and, hence, FSG transitions) + * reachable from that node. + * + * To conserve memory, the underlying HMMs with state-level information are + * allocated only as needed. Root and leaf nodes must also account for all + * the possible phonetic contexts, with an independent HMM for each distinct + * context. + */ +typedef struct fsg_pnode_s { + /* + * If this is not a leaf node, the first successor (child) node. Otherwise + * the parent FSG transition for which this is the leaf node (for figuring + * the FSG destination state, and word emitted by the transition). A node + * may have several children. The succ ptr gives just the first; the rest + * are linked via the sibling ptr below. + */ + union { + struct fsg_pnode_s *succ; + fsg_link_t *fsglink; + } next; + + /* + * For simplicity of memory management (i.e., freeing the pnodes), all + * pnodes allocated for all transitions out of a state are maintained in a + * linear linked list through the alloc_next pointer. + */ + struct fsg_pnode_s *alloc_next; + + /* + * The next node that is also a child of the parent of this node; NULL if + * none. + */ + struct fsg_pnode_s *sibling; + + /* + * The transition (log) probability to be incurred upon transitioning to + * this node. (Transition probabilities are really associated with the + * transitions. But a lextree node has exactly one incoming transition. + * Hence, the prob can be associated with the node.) + * This is a logs2(prob) value, and includes the language weight. + */ + int32 logs2prob; + + /* + * The root and leaf positions associated with any transition have to deal + * with multiple phonetic contexts. However, different contexts may result + * in the same SSID (senone-seq ID), and can share a single pnode with that + * SSID. But the pnode should track the set of context CI phones that share + * it. Hence the fsg_pnode_ctxt_t bit-vector set-representation. (For + * simplicity of implementation, its size is a compile-time constant for + * now.) Single phone words would need a 2-D array of context, but that's + * too expensive. For now, they simply use SIL as right context, so only + * the left context is properly modelled. + * (For word-internal phones, this field is unused, of course.) + */ + fsg_pnode_ctxt_t ctxt; + + uint16 ci_ext; /* This node's CIphone as viewed externally (context) */ + uint8 ppos; /* Phoneme position in pronunciation */ + uint8 leaf; /* Whether this is a leaf node */ + + /* HMM-state-level stuff here */ + hmm_context_t *ctx; + hmm_t hmm; +} fsg_pnode_t; + +/* Access macros */ +#define fsg_pnode_leaf(p) ((p)->leaf) +#define fsg_pnode_logs2prob(p) ((p)->logs2prob) +#define fsg_pnode_succ(p) ((p)->next.succ) +#define fsg_pnode_fsglink(p) ((p)->next.fsglink) +#define fsg_pnode_sibling(p) ((p)->sibling) +#define fsg_pnode_hmmptr(p) (&((p)->hmm)) +#define fsg_pnode_ci_ext(p) ((p)->ci_ext) +#define fsg_pnode_ppos(p) ((p)->ppos) +#define fsg_pnode_leaf(p) ((p)->leaf) +#define fsg_pnode_ctxt(p) ((p)->ctxt) + +#define fsg_pnode_add_ctxt(p,c) ((p)->ctxt.bv[(c)>>5] |= (1 << ((c)&0x001f))) + +/* + * The following is macroized because its called very frequently + * ::: uint32 fsg_pnode_ctxt_sub (fsg_pnode_ctxt_t *src, fsg_pnode_ctxt_t *sub); + */ +/* + * Subtract bitvector sub from bitvector src (src updated with the result). + * Return 0 if result is all 0, non-zero otherwise. + */ + +#if (FSG_PNODE_CTXT_BVSZ == 1) + #define FSG_PNODE_CTXT_SUB(src,sub) \ + ((src)->bv[0] = (~((sub)->bv[0]) & (src)->bv[0])) +#elif (FSG_PNODE_CTXT_BVSZ == 2) + #define FSG_PNODE_CTXT_SUB(src,sub) \ + (((src)->bv[0] = (~((sub)->bv[0]) & (src)->bv[0])) | \ + ((src)->bv[1] = (~((sub)->bv[1]) & (src)->bv[1]))) +#elif (FSG_PNODE_CTXT_BVSZ == 4) + #define FSG_PNODE_CTXT_SUB(src,sub) \ + (((src)->bv[0] = (~((sub)->bv[0]) & (src)->bv[0])) | \ + ((src)->bv[1] = (~((sub)->bv[1]) & (src)->bv[1])) | \ + ((src)->bv[2] = (~((sub)->bv[2]) & (src)->bv[2])) | \ + ((src)->bv[3] = (~((sub)->bv[3]) & (src)->bv[3]))) +#else + #define FSG_PNODE_CTXT_SUB(src,sub) fsg_pnode_ctxt_sub_generic((src),(sub)) +#endif + +/** + * Collection of lextrees for an FSG. + */ +typedef struct fsg_lextree_s { + fsg_model_t *fsg; /**< The fsg for which this lextree is built. */ + hmm_context_t *ctx; /**< HMM context structure. */ + dict_t *dict; /**< Pronunciation dictionary for this FSG. */ + dict2pid_t *d2p; /**< Context-dependent phone mappings for this FSG. */ + bin_mdef_t *mdef; /**< Model definition (triphone mappings). */ + + /* + * Left and right CIphone sets for each state. + * Left context CIphones for a state S: If word W transitions into S, W's + * final CIphone is in S's {lc}. Words transitioning out of S must consider + * these left context CIphones. + * Similarly, right contexts for state S: If word W transitions out of S, + * W's first CIphone is in S's {rc}. Words transitioning into S must consider + * these right contexts. + * + * NOTE: Words may transition into and out of S INDIRECTLY, with intermediate + * null transitions. + * NOTE: Single-phone words are difficult; only SILENCE right context is + * modelled for them. + * NOTE: Non-silence filler phones aren't included in these sets. Filler + * words don't use context, and present the SILENCE phone as context to + * adjacent words. + */ + int16 **lc; /**< Left context triphone mappings for FSG. */ + int16 **rc; /**< Right context triphone mappings for FSG. */ + + fsg_pnode_t **root; /* root[s] = lextree representing all transitions + out of state s. Note that the "tree" for each + state is actually a collection of trees, linked + via fsg_pnode_t.sibling (root[s]->sibling) */ + fsg_pnode_t **alloc_head; /* alloc_head[s] = head of linear list of all + pnodes allocated for state s */ + int32 n_pnode; /* #HMM nodes in search structure */ + int32 wip; + int32 pip; +} fsg_lextree_t; + +/* Access macros */ +#define fsg_lextree_root(lt,s) ((lt)->root[s]) +#define fsg_lextree_n_pnode(lt) ((lt)->n_pnode) + +/** + * Create, initialize, and return a new phonetic lextree for the given FSG. + */ +fsg_lextree_t *fsg_lextree_init(fsg_model_t *fsg, dict_t *dict, + dict2pid_t *d2p, + bin_mdef_t *mdef, hmm_context_t *ctx, + int32 wip, int32 pip); + +/** + * Free lextrees for an FSG. + */ +void fsg_lextree_free(fsg_lextree_t *fsg); + +/** + * Print an FSG lextree to a file for debugging. + */ +void fsg_lextree_dump(fsg_lextree_t *fsg, FILE *fh); + +/** + * Mark the given pnode as inactive (for search). + */ +void fsg_psubtree_pnode_deactivate(fsg_pnode_t *pnode); + +/** + * Set all flags on in the given context bitvector. + */ +void fsg_pnode_add_all_ctxt(fsg_pnode_ctxt_t *ctxt); + +/** + * Generic variant for arbitrary size + */ +uint32 fsg_pnode_ctxt_sub_generic(fsg_pnode_ctxt_t *src, fsg_pnode_ctxt_t *sub); + +#endif diff --git a/include/fsg_search_internal.h b/include/fsg_search_internal.h new file mode 100644 index 0000000..7f31359 --- /dev/null +++ b/include/fsg_search_internal.h @@ -0,0 +1,153 @@ +/* -*- c-basic-offset:4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 1999-2004 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +/* + * fsg_search_internal.h -- Search structures for FSG decoding. + */ + + +#ifndef __S2_FSG_SEARCH_H__ +#define __S2_FSG_SEARCH_H__ + + +/* SphinxBase headers. */ +#include +#include +#include + +/* Local headers. */ +#include "pocketsphinx_internal.h" +#include "hmm.h" +#include "fsg_history.h" +#include "fsg_lextree.h" + +/** + * Segmentation "iterator" for FSG history. + */ +typedef struct fsg_seg_s { + ps_seg_t base; /**< Base structure. */ + fsg_hist_entry_t **hist; /**< Sequence of history entries. */ + int16 n_hist; /**< Number of history entries. */ + int16 cur; /**< Current position in hist. */ +} fsg_seg_t; + +/** + * Implementation of FSG search (and "FSG set") structure. + */ +typedef struct fsg_search_s { + ps_search_t base; + + hmm_context_t *hmmctx; /**< HMM context. */ + + fsg_model_t *fsg; /**< FSG model */ + struct fsg_lextree_s *lextree;/**< Lextree structure for the currently + active FSG */ + struct fsg_history_s *history;/**< For storing the Viterbi search history */ + + glist_t pnode_active; /**< Those active in this frame */ + glist_t pnode_active_next; /**< Those activated for the next frame */ + + int32 beam_orig; /**< Global pruning threshold */ + int32 pbeam_orig; /**< Pruning threshold for phone transition */ + int32 wbeam_orig; /**< Pruning threshold for word exit */ + float32 beam_factor; /**< Dynamic/adaptive factor (<=1) applied to above + beams to determine actual effective beams. + For implementing absolute pruning. */ + int32 beam, pbeam, wbeam; /**< Effective beams after applying beam_factor */ + int32 lw, pip, wip; /**< Language weights */ + + frame_idx_t frame; /**< Current frame. */ + uint8 final; /**< Decoding is finished for this utterance. */ + uint8 bestpath; /**< Whether to run bestpath search + and confidence annotation at end. */ + float32 ascale; /**< Acoustic score scale for posterior probabilities. */ + + int32 bestscore; /**< For beam pruning */ + int32 bpidx_start; /**< First history entry index this frame */ + + int32 ascr, lscr; /**< Total acoustic and lm score for utt */ + + int32 n_hmm_eval; /**< Total HMMs evaluated this utt */ + int32 n_sen_eval; /**< Total senones evaluated this utt */ + + ptmr_t perf; /**< Performance counter */ + int32 n_tot_frame; + +} fsg_search_t; + +/* Access macros */ +#define fsg_search_frame(s) ((s)->frame) + +/** + * Create, initialize and return a search module. + */ +ps_search_t *fsg_search_init(const char *name, + fsg_model_t *fsg, + cmd_ln_t *config, + acmod_t *acmod, + dict_t *dict, + dict2pid_t *d2p); + +/** + * Deallocate search structure. + */ +void fsg_search_free(ps_search_t *search); + +/** + * Update FSG search module for new or updated FSGs. + */ +int fsg_search_reinit(ps_search_t *fsgs, dict_t *dict, dict2pid_t *d2p); + +/** + * Prepare the FSG search structure for beginning decoding of the next + * utterance. + */ +int fsg_search_start(ps_search_t *search); + +/** + * Step one frame forward through the Viterbi search. + */ +int fsg_search_step(ps_search_t *search, int frame_idx); + +/** + * Windup and clean the FSG search structure after utterance. + */ +int fsg_search_finish(ps_search_t *search); + +/** + * Get hypothesis string from the FSG search. + */ +char const *fsg_search_hyp(ps_search_t *search, int32 *out_score); + +#endif diff --git a/include/hmm.h b/include/hmm.h new file mode 100644 index 0000000..2bfb462 --- /dev/null +++ b/include/hmm.h @@ -0,0 +1,306 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 1999-2004 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +/** + * @file hmm.h Hidden Markov Model base structures. + */ + +#ifndef __HMM_H__ +#define __HMM_H__ + +/* System headers. */ +#include + +/* SphinxBase headers. */ +#include +#include + +/* PocketSphinx headers. */ +#include "bin_mdef.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Type for frame index values. Used in HMM indexes and + * backpointers and affects memory required.Due to limitations of FSG + * search implementation this value needs to be signed. + */ +typedef int32 frame_idx_t; + +/** + * Maximum number of frames in index, should be in sync with above. + */ +#define MAX_N_FRAMES MAX_INT32 + + +/** Shift count for senone scores. */ +#define SENSCR_SHIFT 10 + +/** + * Large "bad" score. + * + * This number must be "bad" enough so that 4 times WORST_SCORE will + * not overflow. The reason for this is that the search doesn't check + * the scores in a model before evaluating the model and it may + * require as many was 4 plies before the new 'good' score can wipe + * out the initial WORST_SCORE initialization. + */ +#define WORST_SCORE ((int)0xE0000000) + +/** + * Watch out, though! Transition matrix entries that are supposed to + * be "zero" don't actually get that small due to quantization. + */ +#define TMAT_WORST_SCORE (-255) + +/** + * Is one score better than another? + */ +#define BETTER_THAN > + +/** + * Is one score worse than another? + */ +#define WORSE_THAN < + +/** \file hmm.h + * \brief HMM data structure and operation + * + * For efficiency, this version is hardwired for two possible HMM + * topologies, but will fall back to others: + * + * 5-state left-to-right HMMs: (0 is the *emitting* entry state and E + * is a non-emitting exit state; the x's indicate allowed transitions + * between source and destination states): + * + * 
+ *               0   1   2   3   4   E (destination-states)
+ *           0   x   x   x
+ *           1       x   x   x
+ *           2           x   x   x
+ *           3               x   x   x
+ *           4                   x   x
+ *    (source-states)
+ *
+ * + * 5-state topologies that contain a subset of the above transitions should work as well. + * + * 3-state left-to-right HMMs (similar notation as the 5-state topology above): + * + * 
+ *               0   1   2   E (destination-states)
+ *           0   x   x   x
+ *           1       x   x   x
+ *           2           x   x 
+ *    (source-states)
+ *
+ * + * 3-state topologies that contain a subset of the above transitions should work as well. + */ + +/** + * @struct hmm_context_t + * @brief Shared information between a set of HMMs. + * + * We assume that the initial state is emitting and that the + * transition matrix is n_emit_state x (n_emit_state+1), where the + * extra destination dimension correponds to the non-emitting final or + * exit state. + */ +typedef struct hmm_context_s { + int32 n_emit_state; /**< Number of emitting states in this set of HMMs. */ + uint8 ** const *tp; /**< State transition scores tp[id][from][to] (logs3 values). */ + int16 const *senscore; /**< State emission scores senscore[senid] + (negated scaled logs3 values). */ + uint16 * const *sseq; /**< Senone sequence mapping. */ + int32 *st_sen_scr; /**< Temporary array of senone scores (for some topologies). */ + listelem_alloc_t *mpx_ssid_alloc; /**< Allocator for senone sequence ID arrays. */ + void *udata; /**< Whatever you feel like, gosh. */ +} hmm_context_t; + +/** + * Hard-coded limit on the number of emitting states. + */ +#define HMM_MAX_NSTATE 5 + +/** + * @struct hmm_t + * @brief An individual HMM among the HMM search space. + * + * An individual HMM among the HMM search space. An HMM with N + * emitting states consists of N+1 internal states including the + * non-emitting exit (out) state. + */ +typedef struct hmm_s { + hmm_context_t *ctx; /**< Shared context data for this HMM. */ + int32 score[HMM_MAX_NSTATE]; /**< State scores for emitting states. */ + int32 history[HMM_MAX_NSTATE]; /**< History indices for emitting states. */ + int32 out_score; /**< Score for non-emitting exit state. */ + int32 out_history; /**< History index for non-emitting exit state. */ + uint16 ssid; /**< Senone sequence ID (for non-MPX) */ + uint16 senid[HMM_MAX_NSTATE]; /**< Senone IDs (non-MPX) or sequence IDs (MPX) */ + int32 bestscore; /**< Best [emitting] state score in current frame (for pruning). */ + int16 tmatid; /**< Transition matrix ID (see hmm_context_t). */ + frame_idx_t frame; /**< Frame in which this HMM was last active; <0 if inactive */ + uint8 mpx; /**< Is this HMM multiplex? (hoisted for speed) */ + uint8 n_emit_state; /**< Number of emitting states (hoisted for speed) */ +} hmm_t; + +/** Access macros. */ +#define hmm_context(h) (h)->ctx +#define hmm_is_mpx(h) (h)->mpx + +#define hmm_in_score(h) (h)->score[0] +#define hmm_score(h,st) (h)->score[st] +#define hmm_out_score(h) (h)->out_score + +#define hmm_in_history(h) (h)->history[0] +#define hmm_history(h,st) (h)->history[st] +#define hmm_out_history(h) (h)->out_history + +#define hmm_bestscore(h) (h)->bestscore +#define hmm_frame(h) (h)->frame +#define hmm_mpx_ssid(h,st) (h)->senid[st] +#define hmm_nonmpx_ssid(h) (h)->ssid +#define hmm_ssid(h,st) (hmm_is_mpx(h) \ + ? hmm_mpx_ssid(h,st) : hmm_nonmpx_ssid(h)) +#define hmm_mpx_senid(h,st) (hmm_mpx_ssid(h,st) == BAD_SENID \ + ? BAD_SENID : (h)->ctx->sseq[hmm_mpx_ssid(h,st)][st]) +#define hmm_nonmpx_senid(h,st) ((h)->senid[st]) +#define hmm_senid(h,st) (hmm_is_mpx(h) \ + ? hmm_mpx_senid(h,st) : hmm_nonmpx_senid(h,st)) +#define hmm_senscr(h,st) (hmm_senid(h,st) == BAD_SENID \ + ? WORST_SCORE \ + : -(h)->ctx->senscore[hmm_senid(h,st)]) +#define hmm_tmatid(h) (h)->tmatid +#define hmm_tprob(h,i,j) (-(h)->ctx->tp[hmm_tmatid(h)][i][j]) +#define hmm_n_emit_state(h) ((h)->n_emit_state) +#define hmm_n_state(h) ((h)->n_emit_state + 1) + +/** + * Create an HMM context. + **/ +hmm_context_t *hmm_context_init(int32 n_emit_state, + uint8 ** const *tp, + int16 const *senscore, + uint16 * const *sseq); + +/** + * Change the senone score array for a context. + **/ +#define hmm_context_set_senscore(ctx, senscr) ((ctx)->senscore = (senscr)) + +/** + * Free an HMM context. + * + * @note The transition matrices, senone scores, and senone sequence + * mapping are all assumed to be allocated externally, and will NOT be + * freed by this function. + **/ +void hmm_context_free(hmm_context_t *ctx); + +/** + * Populate a previously-allocated HMM structure, allocating internal data. + **/ +void hmm_init(hmm_context_t *ctx, hmm_t *hmm, int mpx, int ssid, int tmatid); + +/** + * Free an HMM structure, releasing internal data (but not the HMM structure itself). + */ +void hmm_deinit(hmm_t *hmm); + +/** + * Reset the states of the HMM to the invalid condition. + + * i.e., scores to WORST_SCORE and hist to undefined. + */ +void hmm_clear(hmm_t *h); + +/** + * Reset the scores of the HMM. + */ +void hmm_clear_scores(hmm_t *h); + +/** + * Renormalize the scores in this HMM based on the given best score. + */ +void hmm_normalize(hmm_t *h, int32 bestscr); + +/** + * Enter an HMM with the given path score and history ID. + **/ +void hmm_enter(hmm_t *h, int32 score, + int32 histid, int frame); + +/** + * Viterbi evaluation of given HMM. + * + * @note If this module were being used for tracking state + * segmentations, the dummy, non-emitting exit state would have to be + * updated separately. In the Viterbi DP diagram, transitions to the + * exit state occur from the current time; they are vertical + * transitions. Hence they should be made only after the history has + * been logged for the emitting states. But we're not bothered with + * state segmentations, for now. So, we update the exit state as + * well. +*/ +int32 hmm_vit_eval(hmm_t *hmm); + + +/** + * Like hmm_vit_eval, but dump HMM state and relevant senscr to fp first, for debugging;. + */ +int32 hmm_dump_vit_eval(hmm_t *hmm, /**< In/Out: HMM being updated */ + FILE *fp /**< An output file pointer */ + ); + +/** + * For debugging, dump the whole HMM out. + */ + +void hmm_dump(hmm_t *h, /**< In/Out: HMM being updated */ + FILE *fp /**< An output file pointer */ + ); + + +#ifdef __cplusplus +} +#endif + +#endif /* __HMM_H__ */ diff --git a/include/kws_detections.h b/include/kws_detections.h new file mode 100644 index 0000000..855c3bc --- /dev/null +++ b/include/kws_detections.h @@ -0,0 +1,76 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 2014 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +/* + * kws_detections.h -- Structures for storing keyphrase spotting results. + */ + +#ifndef __KWS_DETECTIONS_H__ +#define __KWS_DETECTIONS_H__ + +/* SphinxBase headers. */ +#include + +/* Local headers. */ +#include "pocketsphinx_internal.h" +#include "hmm.h" + +typedef struct kws_detection_s { + const char* keyphrase; + frame_idx_t sf; + frame_idx_t ef; + int32 prob; + int32 ascr; +} kws_detection_t; + +typedef struct kws_detections_s { + glist_t detect_list; +} kws_detections_t; + +/** + * Reset history structure. + */ +void kws_detections_reset(kws_detections_t *detections); + +/** + * Add history entry. + */ +void kws_detections_add(kws_detections_t *detections, const char* keyphrase, int sf, int ef, int prob, int ascr); + +/** + * Compose hypothesis. + */ +char* kws_detections_hyp_str(kws_detections_t *detections, int frame, int delay); + +#endif /* __KWS_DETECTIONS_H__ */ diff --git a/include/kws_search.h b/include/kws_search.h new file mode 100644 index 0000000..c820afb --- /dev/null +++ b/include/kws_search.h @@ -0,0 +1,142 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 2013 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +/* + * kws_search.h -- Search structures for keyphrase spotting. + */ + +#ifndef __KWS_SEARCH_H__ +#define __KWS_SEARCH_H__ + +/* SphinxBase headers. */ +#include +#include + +/* Local headers. */ +#include "pocketsphinx_internal.h" +#include "kws_detections.h" +#include "hmm.h" + +/** + * Segmentation "iterator" for KWS history. + */ +typedef struct kws_seg_s { + ps_seg_t base; /**< Base structure. */ + gnode_t *detection; /**< Keyphrase detection correspondent to segment. */ + frame_idx_t last_frame; /**< Last frame to raise the detection */ +} kws_seg_t; + +typedef struct kws_keyphrase_s { + char* word; + int32 threshold; + hmm_t* hmms; + int32 n_hmms; +} kws_keyphrase_t; + +/** + * Implementation of KWS search structure. + */ +typedef struct kws_search_s { + ps_search_t base; + + hmm_context_t *hmmctx; /**< HMM context. */ + + glist_t keyphrases; /**< Keyphrases to spot */ + + kws_detections_t *detections; /**< Keyword spotting history */ + frame_idx_t frame; /**< Frame index */ + + int32 beam; + + int32 plp; /**< Phone loop probability */ + int32 bestscore; /**< For beam pruning */ + int32 def_threshold; /**< default threshold for p(hyp)/p(altern) ratio */ + int32 delay; /**< Delay to wait for best detection score */ + + int32 n_pl; /**< Number of CI phones */ + hmm_t *pl_hmms; /**< Phone loop hmms - hmms of CI phones */ + + ptmr_t perf; /**< Performance counter */ + int32 n_tot_frame; + +} kws_search_t; + +/** + * Create, initialize and return a search module. Gets keyphrases either + * from keyphrase or from a keyphrase file. + */ +ps_search_t *kws_search_init(const char *name, + const char *keyphrase, + const char *keyfile, + cmd_ln_t * config, + acmod_t * acmod, + dict_t * dict, dict2pid_t * d2p); + +/** + * Deallocate search structure. + */ +void kws_search_free(ps_search_t * search); + +/** + * Update KWS search module for new key phrase. + */ +int kws_search_reinit(ps_search_t * kwss, dict_t * dict, dict2pid_t * d2p); + +/** + * Prepare the KWS search structure for beginning decoding of the next + * utterance. + */ +int kws_search_start(ps_search_t * search); + +/** + * Step one frame forward through the Viterbi search. + */ +int kws_search_step(ps_search_t * search, int frame_idx); + +/** + * Windup and clean the KWS search structure after utterance. + */ +int kws_search_finish(ps_search_t * search); + +/** + * Get hypothesis string from the KWS search. + */ +char const *kws_search_hyp(ps_search_t * search, int32 * out_score); + +/** + * Get active keyphrases + */ +char* kws_search_get_keyphrases(ps_search_t * search); + +#endif /* __KWS_SEARCH_H__ */ diff --git a/include/mdef.h b/include/mdef.h new file mode 100644 index 0000000..b0a7ced --- /dev/null +++ b/include/mdef.h @@ -0,0 +1,271 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 1999-2004 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +/* + * mdef.h -- HMM model definition: base (CI) phones and triphones + * + * ********************************************** + * CMU ARPA Speech Project + * + * Copyright (c) 1999 Carnegie Mellon University. + * ALL RIGHTS RESERVED. + * ********************************************** + */ + + +#ifndef __MDEF_H__ +#define __MDEF_H__ + + +/* System headers. */ +#include + +/* SphinxBase headers. */ +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** \file mdef.h + * \brief Model definition + */ + +/** \enum word_posn_t + * \brief Union of different type of word position + */ + +typedef enum { + WORD_POSN_INTERNAL = 0, /**< Internal phone of word */ + WORD_POSN_BEGIN = 1, /**< Beginning phone of word */ + WORD_POSN_END = 2, /**< Ending phone of word */ + WORD_POSN_SINGLE = 3, /**< Single phone word (i.e. begin & end) */ + WORD_POSN_UNDEFINED = 4 /**< Undefined value, used for initial conditions, etc */ +} word_posn_t; +#define N_WORD_POSN 4 /**< total # of word positions (excluding undefined) */ +#define WPOS_NAME "ibesu" /**< Printable code for each word position above */ +#define S3_SILENCE_CIPHONE "SIL" /**< Hard-coded silence CI phone name */ + +/** + \struct ciphone_t + \brief CI phone information +*/ +typedef struct { + char *name; /**< The name of the CI phone */ + int32 filler; /**< Whether a filler phone; if so, can be substituted by + silence phone in left or right context position */ +} ciphone_t; + +/** + * \struct phone_t + * \brief Triphone information, including base phones as a subset. For the latter, lc, rc and wpos are non-existent. + */ +typedef struct { + int32 ssid; /**< State sequence (or senone sequence) ID, considering the + n_emit_state senone-ids are a unit. The senone sequences + themselves are in a separate table */ + int32 tmat; /**< Transition matrix id */ + int16 ci, lc, rc; /**< Base, left, right context ciphones */ + word_posn_t wpos; /**< Word position */ + +} phone_t; + +/** + * \struct ph_rc_t + * \brief Structures needed for mapping into pid. (See mdef_t.wpos_ci_lclist below.) (lc = left context; rc = right context.) + * NOTE: Both ph_rc_t and ph_lc_t FOR INTERNAL USE ONLY. + */ +typedef struct ph_rc_s { + int16 rc; /**< Specific rc for a parent */ + int32 pid; /**< Triphone id for above rc instance */ + struct ph_rc_s *next; /**< Next rc entry for same parent */ +} ph_rc_t; + +/** + * \struct ph_lc_t + * \brief Structures for storing the left context. + */ + +typedef struct ph_lc_s { + int16 lc; /**< Specific lc for a parent */ + ph_rc_t *rclist; /**< rc list for above lc instance */ + struct ph_lc_s *next; /**< Next lc entry for same parent */ +} ph_lc_t; + + +/** The main model definition structure */ +/** + \struct mdef_t + \brief strcture for storing the model definition. +*/ +typedef struct { + int32 n_ciphone; /**< number basephones actually present */ + int32 n_phone; /**< number basephones + number triphones actually present */ + int32 n_emit_state; /**< number emitting states per phone */ + int32 n_ci_sen; /**< number CI senones; these are the first */ + int32 n_sen; /**< number senones (CI+CD) */ + int32 n_tmat; /**< number transition matrices */ + + hash_table_t *ciphone_ht; /**< Hash table for mapping ciphone strings to ids */ + ciphone_t *ciphone; /**< CI-phone information for all ciphones */ + phone_t *phone; /**< Information for all ciphones and triphones */ + uint16 **sseq; /**< Unique state (or senone) sequences in this model, shared + among all phones/triphones */ + int32 n_sseq; /**< No. of unique senone sequences in this model */ + + int16 *cd2cisen; /**< Parent CI-senone id for each senone; the first + n_ci_sen are identity mappings; the CD-senones are + contiguous for each parent CI-phone */ + int16 *sen2cimap; /**< Parent CI-phone for each senone (CI or CD) */ + + int16 sil; /**< SILENCE_CIPHONE id */ + + ph_lc_t ***wpos_ci_lclist; /**< wpos_ci_lclist[wpos][ci] = list of lc for . + wpos_ci_lclist[wpos][ci][lc].rclist = list of rc for + . Only entries for the known triphones + are created to conserve space. + (NOTE: FOR INTERNAL USE ONLY.) */ +} mdef_t; + +/** Access macros; not meant for arbitrary use */ +#define mdef_is_fillerphone(m,p) ((m)->ciphone[p].filler) +#define mdef_n_ciphone(m) ((m)->n_ciphone) +#define mdef_n_phone(m) ((m)->n_phone) +#define mdef_n_sseq(m) ((m)->n_sseq) +#define mdef_n_emit_state(m) ((m)->n_emit_state) +#define mdef_n_sen(m) ((m)->n_sen) +#define mdef_n_tmat(m) ((m)->n_tmat) +#define mdef_pid2ssid(m,p) ((m)->phone[p].ssid) +#define mdef_pid2tmatid(m,p) ((m)->phone[p].tmat) +#define mdef_silphone(m) ((m)->sil) +#define mdef_sen2cimap(m) ((m)->sen2cimap) +#define mdef_sseq2sen(m,ss,pos) ((m)->sseq[ss][pos]) +#define mdef_pid2ci(m,p) ((m)->phone[p].ci) +#define mdef_cd2cisen(m) ((m)->cd2cisen) + +/** + * Initialize the phone structure from the given model definition file. + * It should be treated as a READ-ONLY structure. + * @return pointer to the phone structure created. + */ +mdef_t *mdef_init (char *mdeffile, /**< In: Model definition file */ + int breport /**< In: whether to report the progress or not */ + ); + + +/** + Get the ciphone id given a string name + @return ciphone id for the given ciphone string name +*/ +int mdef_ciphone_id(mdef_t *m, /**< In: Model structure being queried */ + char *ciphone /**< In: ciphone for which id wanted */ + ); + +/** + Get the phone string given the ci phone id. + @return: READ-ONLY ciphone string name for the given ciphone id +*/ +const char *mdef_ciphone_str(mdef_t *m, /**< In: Model structure being queried */ + int ci /**< In: ciphone id for which name wanted */ + ); + +/** + Decide whether the phone is ci phone. + @return 1 if given triphone argument is a ciphone, 0 if not, -1 if error +*/ +int mdef_is_ciphone (mdef_t *m, /**< In: Model structure being queried */ + int p /**< In: triphone id being queried */ + ); + +/** + Decide whether the senone is a senone for a ci phone, or a ci senone + @return 1 if a given senone is a ci senone +*/ +int mdef_is_cisenone(mdef_t *m, /**< In: Model structure being queried */ + int s /**< In: senone id being queried */ + ); + +/** + Decide the phone id given the left, right and base phones. + @return: phone id for the given constituents if found, else BAD_S3PID +*/ +int mdef_phone_id (mdef_t *m, /**< In: Model structure being queried */ + int b, /**< In: base ciphone id */ + int l, /**< In: left context ciphone id */ + int r, /**< In: right context ciphone id */ + word_posn_t pos /**< In: Word position */ + ); + +/** + * Create a phone string for the given phone (base or triphone) id in the given buf. + * @return 0 if successful, -1 if error. + */ +int mdef_phone_str(mdef_t *m, /**< In: Model structure being queried */ + int pid, /**< In: phone id being queried */ + char *buf /**< Out: On return, buf has the string */ + ); + +/** + * Compare the underlying HMMs for two given phones (i.e., compare the two transition + * matrix IDs and the individual state(senone) IDs). + * @return 0 iff the HMMs are identical, -1 otherwise. + */ +int mdef_hmm_cmp (mdef_t *m, /**< In: Model being queried */ + int p1, /**< In: One of the two triphones being compared */ + int p2 /**< In: One of the two triphones being compared */ + ); + +/** Report the model definition's parameters */ +void mdef_report(mdef_t *m /**< In: model definition structure */ + ); + +/** RAH, For freeing memory */ +void mdef_free_recursive_lc (ph_lc_t *lc /**< In: A list of left context */ + ); +void mdef_free_recursive_rc (ph_rc_t *rc /**< In: A list of right context */ + ); + +/** Free an mdef_t */ +void mdef_free (mdef_t *mdef /**< In : The model definition*/ + ); + + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/ms_gauden.h b/include/ms_gauden.h new file mode 100644 index 0000000..9c26cef --- /dev/null +++ b/include/ms_gauden.h @@ -0,0 +1,150 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 1999-2004 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +#ifndef _LIBFBS_GAUDEN_H_ +#define _LIBFBS_GAUDEN_H_ + +/** \file ms_gauden.h + * \brief (Sphinx 3.0 specific) Gaussian density module. + * + * Gaussian density distribution implementation. There are two major + * difference bettwen ms_gauden and cont_mgau. One is the fact that + * ms_gauden only take cares of the Gaussian computation part where + * cont_mgau actually take care of senone computation as well. The + * other is the fact that ms_gauden is a multi-stream implementation + * of GMM computation. + * + */ + +/* SphinxBase headers. */ +#include +#include +#include + +/* Local headers. */ +#include "vector.h" +#include "pocketsphinx_internal.h" +#include "hmm.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * \struct gauden_dist_t + * \brief Structure to store distance (density) values for a given input observation wrt density values in some given codebook. + */ +typedef struct { + int32 id; /**< Index of codeword (gaussian density) */ + mfcc_t dist; /**< Density value for input observation wrt above codeword; + NOTE: result in logs3 domain, but var_t used for speed */ + +} gauden_dist_t; + +/** + * \struct gauden_t + * \brief Multivariate gaussian mixture density parameters + */ +typedef struct { + mfcc_t ****mean; /**< mean[codebook][feature][codeword] vector */ + mfcc_t ****var; /**< like mean; diagonal covariance vector only */ + mfcc_t ***det; /**< log(determinant) for each variance vector; + actually, log(sqrt(2*pi*det)) */ + logmath_t *lmath; /**< log math computation */ + int32 n_mgau; /**< Number codebooks */ + int32 n_feat; /**< Number feature streams in each codebook */ + int32 n_density; /**< Number gaussian densities in each codebook-feature stream */ + int32 *featlen; /**< feature length for each feature */ +} gauden_t; + + +/** + * Read mixture gaussian codebooks from the given files. Allocate memory space needed + * for them. Apply the specified variance floor value. + * Return value: ptr to the model created; NULL if error. + * (See Sphinx3 model file-format documentation.) + */ +gauden_t * +gauden_init (char const *meanfile,/**< Input: File containing means of mixture gaussians */ + char const *varfile,/**< Input: File containing variances of mixture gaussians */ + float32 varfloor, /**< Input: Floor value to be applied to variances */ + logmath_t *lmath + ); + +/** Release memory allocated by gauden_init. */ +void gauden_free(gauden_t *g); /**< In: The gauden_t to free */ + +/** Transform Gaussians according to an MLLR matrix (or, eventually, more). */ +int32 gauden_mllr_transform(gauden_t *s, ps_mllr_t *mllr, cmd_ln_t *config); + +/** + * Compute gaussian density values for the given input observation vector wrt the + * specified mixture gaussian codebook (which may consist of several feature streams). + * Density values are left UNnormalized. + * @return 0 if successful, -1 otherwise. + */ +int32 +gauden_dist (gauden_t *g, /**< In: handle to entire ensemble of codebooks */ + int mgau, /**< In: codebook for which density values to be evaluated + (g->{mean,var}[mgau]) */ + int n_top, /**< In: Number top densities to be evaluated */ + mfcc_t **obs, /**< In: Observation vector; obs[f] = for feature f */ + gauden_dist_t **out_dist + /**< Out: n_top best codewords and density values, + in worsening order, for each feature stream. + out_dist[f][i] = i-th best density for feature f. + Caller must allocate memory for this output */ + ); + +/** + Dump the definitionn of Gaussian distribution. +*/ +void gauden_dump (const gauden_t *g /**< In: Gaussian distribution g*/ + ); + +/** + Dump the definition of Gaussian distribution of a particular index to the standard output stream +*/ +void gauden_dump_ind (const gauden_t *g, /**< In: Gaussian distribution g*/ + int senidx /**< In: The senone index of the Gaussian */ + ); + +#ifdef __cplusplus +} +#endif + +#endif /* GAUDEN_H */ diff --git a/include/ms_mgau.h b/include/ms_mgau.h new file mode 100644 index 0000000..b018dcc --- /dev/null +++ b/include/ms_mgau.h @@ -0,0 +1,143 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 1999-2004 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ +/* + * ms_mgau.h -- Essentially a wrapper that wrap up gauden and + * senone. It supports multi-stream. + * + * + * ********************************************** + * CMU ARPA Speech Project + * + * Copyright (c) 1997 Carnegie Mellon University. + * ALL RIGHTS RESERVED. + * ********************************************** + * HISTORY + * $Log$ + * Revision 1.1 2006/04/05 20:27:30 dhdfu + * A Great Reorganzation of header files and executables + * + * Revision 1.3 2006/02/22 16:57:15 arthchan2003 + * Fixed minor dox-doc issue + * + * Revision 1.2 2006/02/22 16:56:01 arthchan2003 + * Merged from SPHINX3_5_2_RCI_IRII_BRANCH: Added ms_mgau.[ch] into the trunk. It is a wrapper of ms_gauden and ms_senone + * + * Revision 1.1.2.4 2005/09/25 18:55:19 arthchan2003 + * Added a flag to turn on and off precomputation. + * + * Revision 1.1.2.3 2005/08/03 18:53:44 dhdfu + * Add memory deallocation functions. Also move all the initialization + * of ms_mgau_model_t into ms_mgau_init (duh!), which entails removing it + * from decode_anytopo and friends. + * + * Revision 1.1.2.2 2005/08/02 21:05:38 arthchan2003 + * 1, Added dist and mgau_active as intermediate variable for computation. 2, Added ms_cont_mgau_frame_eval, which is a multi stream version of GMM computation mainly s3.0 family of tools. 3, Fixed dox-doc. + * + * Revision 1.1.2.1 2005/07/20 19:37:09 arthchan2003 + * Added a multi-stream cont_mgau (ms_mgau) which is a wrapper of both gauden and senone. Add ms_mgau_init and model_set_mllr. This allow eliminating 600 lines of code in decode_anytopo/align/allphone. + * + * + * + */ + +/** \file ms_mgau.h + * + * \brief (Sphinx 3.0 specific) A module that wraps up the code of + * gauden and senone because they are closely related. + * + * At the time at Sphinx 3.1 to 3.2, Ravi has decided to rewrite only + * single-stream part of the code into cont_mgau.[ch]. This marks the + * beginning of historical problem of having two sets of Gaussian + * distribution computation routine, one for single-stream and one of + * multi-stream. + * + * In Sphinx 3.5, when we figure out that it is possible to allow both + * 3.0 family of tools and 3.x family of tools to coexist. This + * becomes one problem we found that very hard to reconcile. That is + * why we currently allow two versions of the code in the code + * base. This is likely to change in the future. + */ + + +#ifndef _LIBFBS_MS_CONT_MGAU_H_ +#define _LIBFBS_MS_CONT_MGAU_H_ + +/* SphinxBase headers. */ +#include +#include +#include + +/* Local headers. */ +#include "acmod.h" +#include "bin_mdef.h" +#include "ms_gauden.h" +#include "ms_senone.h" + +/** \struct ms_mgau_t + \brief Multi-stream mixture gaussian. It is not necessary to be continr +*/ + +typedef struct { + ps_mgau_t base; + gauden_t* g; /**< The codebook */ + senone_t* s; /**< The senone */ + int topn; /**< Top-n gaussian will be computed */ + + /**< Intermediate used in computation */ + gauden_dist_t ***dist; + uint8 *mgau_active; + cmd_ln_t *config; +} ms_mgau_model_t; + +#define ms_mgau_gauden(msg) (msg->g) +#define ms_mgau_senone(msg) (msg->s) +#define ms_mgau_topn(msg) (msg->topn) + +ps_mgau_t* ms_mgau_init(acmod_t *acmod, logmath_t *lmath, bin_mdef_t *mdef); +void ms_mgau_free(ps_mgau_t *g); +int32 ms_cont_mgau_frame_eval(ps_mgau_t * msg, + int16 *senscr, + uint8 *senone_active, + int32 n_senone_active, + mfcc_t ** feat, + int32 frame, + int32 compallsen); +int32 ms_mgau_mllr_transform(ps_mgau_t *s, + ps_mllr_t *mllr); + +#endif /* _LIBFBS_MS_CONT_MGAU_H_*/ + diff --git a/include/ms_senone.h b/include/ms_senone.h new file mode 100644 index 0000000..d92b638 --- /dev/null +++ b/include/ms_senone.h @@ -0,0 +1,131 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 1999-2004 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ +/* + * senone.h -- Mixture density weights associated with each tied state. + */ + +#ifndef _MS_SENONE_H_ +#define _MS_SENONE_H_ + + +/* SphinxBase headers. */ +#include +#include +#include +#include + +/* Local headers. */ +#include "ms_gauden.h" +#include "bin_mdef.h" + +/** \file ms_senone.h + * \brief (Sphinx 3.0 specific) multiple streams senones. used with ms_gauden.h + * In Sphinx 3.0 family of tools, ms_senone is used to combine the Gaussian scores. + * Its existence is crucial in Sphinx 3.0 because 3.0 supports both SCHMM and CDHMM. + * There are optimization scheme for SCHMM (e.g. compute the top-N Gaussian) that is + * applicable to SCHMM than CDHMM. This is wrapped in senone_eval_all. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef uint8 senprob_t; /**< Senone logs3-probs, truncated to 8 bits */ + +/** + * \struct senone_t + * \brief 8-bit senone PDF structure. + * + * 8-bit senone PDF structure. Senone pdf values are normalized, floored, converted to + * logs3 domain, and finally truncated to 8 bits precision to conserve memory space. + */ +typedef struct { + senprob_t ***pdf; /**< gaussian density mixture weights, organized two possible + ways depending on n_gauden: + if (n_gauden > 1): pdf[sen][feat][codeword]. Not an + efficient representation--memory access-wise--but + evaluating the many codebooks will be more costly. + if (n_gauden == 1): pdf[feat][codeword][sen]. Optimized + for the shared-distribution semi-continuous case. */ + logmath_t *lmath; /**< log math computation */ + uint32 n_sen; /**< Number senones in this set */ + uint32 n_feat; /**< Number feature streams */ + uint32 n_cw; /**< Number codewords per codebook,stream */ + uint32 n_gauden; /**< Number gaussian density codebooks referred to by senones */ + float32 mixwfloor; /**< floor applied to each PDF entry */ + uint32 *mgau; /**< senone-id -> mgau-id mapping for senones in this set */ + int32 *featscr; /**< The feature score for every senone, will be initialized inside senone_eval_all */ + int32 aw; /**< Inverse acoustic weight */ +} senone_t; + + +/** + * Load a set of senones (mixing weights and mixture gaussian codebook mappings) from + * the given files. Normalize weights for each codebook, apply the given floor, convert + * PDF values to logs3 domain and quantize to 8-bits. + * @return pointer to senone structure created. Caller MUST NOT change its contents. + */ +senone_t *senone_init (gauden_t *g, /**< In: codebooks */ + char const *mixwfile, /**< In: mixing weights file */ + char const *mgau_mapfile,/**< In: file specifying mapping from each + senone to mixture gaussian codebook. + If NULL all senones map to codebook 0 */ + float32 mixwfloor, /**< In: Floor value for senone weights */ + logmath_t *lmath, /**< In: log math computation */ + bin_mdef_t *mdef /**< In: model definition */ + ); + +/** Release memory allocated by senone_init. */ +void senone_free(senone_t *s); /**< In: The senone_t to free */ + +/** + * Evaluate the score for the given senone wrt to the given top N gaussian codewords. + * @return senone score (in logs3 domain). + */ +int32 senone_eval (senone_t *s, int id, /**< In: senone for which score desired */ + gauden_dist_t **dist, /**< In: top N codewords and densities for + all features, to be combined into + senone score. IE, dist[f][i] = i-th + best for feaure f */ + int n_top /**< In: Length of dist[f], for each f */ + ); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/ngram_search.h b/include/ngram_search.h new file mode 100644 index 0000000..fe0a98d --- /dev/null +++ b/include/ngram_search.h @@ -0,0 +1,434 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 2008 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +/** + * @file ngram_search.h N-Gram based multi-pass search ("FBS") + */ + +#ifndef __NGRAM_SEARCH_H__ +#define __NGRAM_SEARCH_H__ + +/* SphinxBase headers. */ +#include +#include +#include +#include +#include + +/* Local headers. */ +#include "pocketsphinx_internal.h" +#include "hmm.h" + +/** + * Lexical tree node data type. + * + * Not the first HMM for words, which multiplex HMMs based on + * different left contexts. This structure is used both in the + * dynamic HMM tree structure and in the per-word last-phone right + * context fanout. + */ +typedef struct chan_s { + hmm_t hmm; /**< Basic HMM structure. This *must* be first in + the structure because chan_t and root_chan_t are + sometimes used interchangeably */ + struct chan_s *next; /**< first descendant of this channel; or, in the + case of the last phone of a word, the next + alternative right context channel */ + struct chan_s *alt; /**< sibling; i.e., next descendant of parent HMM */ + + int32 ciphone; /**< ciphone for this node */ + union { + int32 penult_phn_wid; /**< list of words whose last phone follows this one; + this field indicates the first of the list; the + rest must be built up in a separate array. Used + only within HMM tree. -1 if none */ + int32 rc_id; /**< right-context id for last phone of words */ + } info; +} chan_t; + +/** + * Lexical tree node data type for the first phone (root) of each dynamic HMM tree + * structure. + * + * Each state may have a different parent static HMM. Most fields are + * similar to those in chan_t. + */ +typedef struct root_chan_s { + hmm_t hmm; /**< Basic HMM structure. This *must* be first in + the structure because chan_t and root_chan_t are + sometimes used interchangeably. */ + chan_t *next; /**< first descendant of this channel */ + + int32 penult_phn_wid; + int32 this_phn_wid; /**< list of words consisting of this single phone; + actually the first of the list, like penult_phn_wid; + -1 if none */ + int16 ciphone; /**< first ciphone of this node; all words rooted at this + node begin with this ciphone */ + int16 ci2phone; /**< second ciphone of this node; one root HMM for each + unique right context */ +} root_chan_t; + +/** + * Back pointer table (forward pass lattice; actually a tree) + */ +typedef struct bptbl_s { + frame_idx_t frame; /**< start or end frame */ + uint8 valid; /**< For absolute pruning */ + uint8 refcnt; /**< Reference count (number of successors) */ + int32 wid; /**< Word index */ + int32 bp; /**< Back Pointer */ + int32 score; /**< Score (best among all right contexts) */ + int32 s_idx; /**< Start of BScoreStack for various right contexts*/ + int32 real_wid; /**< wid of this or latest predecessor real word */ + int32 prev_real_wid; /**< wid of second-last real word */ + int16 last_phone; /**< last phone of this word */ + int16 last2_phone; /**< next-to-last phone of this word */ +} bptbl_t; + +/** + * Segmentation "iterator" for backpointer table results. + */ +typedef struct bptbl_seg_s { + ps_seg_t base; /**< Base structure. */ + int32 *bpidx; /**< Sequence of backpointer IDs. */ + int16 n_bpidx; /**< Number of backpointer IDs. */ + int16 cur; /**< Current position in bpidx. */ +} bptbl_seg_t; + +/* + * Candidates words for entering their last phones. Cleared and rebuilt in each + * frame. + * NOTE: candidates can only be multi-phone, real dictionary words. + */ +typedef struct lastphn_cand_s { + int32 wid; + int32 score; + int32 bp; + int32 next; /* next candidate starting at the same frame */ +} lastphn_cand_t; + +/* + * Since the same instance of a word (i.e., ) reaches its last + * phone several times, we can compute its best BP and LM transition score info + * just the first time and cache it for future occurrences. Structure for such + * a cache. + */ +typedef struct { + int32 sf; /* Start frame */ + int32 dscr; /* Delta-score upon entering last phone */ + int32 bp; /* Best BP */ +} last_ltrans_t; + +#define CAND_SF_ALLOCSIZE 32 +typedef struct { + int32 bp_ef; + int32 cand; +} cand_sf_t; + +/* + * Structure for reorganizing the BP table entries in the current frame according + * to distinct right context ci-phones. Each entry contains the best BP entry for + * a given right context. Each successor word will pick up the correct entry based + * on its first ci-phone. + */ +typedef struct bestbp_rc_s { + int32 score; + int32 path; /* BP table index corresponding to this entry */ + int32 lc; /* right most ci-phone of above BP entry word */ +} bestbp_rc_t; + +#define NO_BP -1 + +/** + * Various statistics for profiling. + */ +typedef struct ngram_search_stats_s { + int32 n_phone_eval; + int32 n_root_chan_eval; + int32 n_nonroot_chan_eval; + int32 n_last_chan_eval; + int32 n_word_lastchan_eval; + int32 n_lastphn_cand_utt; + int32 n_fwdflat_chan; + int32 n_fwdflat_words; + int32 n_fwdflat_word_transition; + int32 n_senone_active_utt; +} ngram_search_stats_t; + + +/** + * N-Gram search module structure. + */ +struct ngram_search_s { + ps_search_t base; + ngram_model_t *lmset; /**< Set of language models. */ + hmm_context_t *hmmctx; /**< HMM context. */ + + /* Flags to quickly indicate which passes are enabled. */ + uint8 fwdtree; + uint8 fwdflat; + uint8 bestpath; + + /* State of procesing. */ + uint8 done; + + /* Allocators */ + listelem_alloc_t *chan_alloc; /**< For chan_t */ + listelem_alloc_t *root_chan_alloc; /**< For root_chan_t */ + listelem_alloc_t *latnode_alloc; /**< For latnode_t */ + + /** + * Search structure of HMM instances. + * + * The word triphone sequences (HMM instances) are transformed + * into tree structures, one tree per unique left triphone in the + * entire dictionary (actually diphone, since its left context + * varies dyamically during the search process). The entire set + * of trees of channels is allocated once and for all during + * initialization (since dynamic management of active CHANs is + * time consuming), with one exception: the last phones of words, + * that need multiple right context modelling, are not maintained + * in this static structure since there are too many of them and + * few are active at any time. Instead they are maintained as + * linked lists of CHANs, one list per word, and each CHAN in this + * set is allocated only on demand and freed if inactive. + */ + root_chan_t *root_chan; /**< Roots of search tree. */ + int32 n_root_chan_alloc; /**< Number of root_chan allocated */ + int32 n_root_chan; /**< Number of valid root_chan */ + int32 n_nonroot_chan; /**< Number of valid non-root channels */ + int32 max_nonroot_chan; /**< Maximum possible number of non-root channels */ + root_chan_t *rhmm_1ph; /**< Root HMMs for single-phone words */ + + /** + * Channels associated with a given word (only used for right + * contexts, single-phone words in fwdtree search, and word HMMs + * in fwdflat search). WARNING: For single-phone words and + * fwdflat search, this actually contains pointers to root_chan_t, + * which are allocated using root_chan_alloc. This is a + * suboptimal state of affairs. + */ + chan_t **word_chan; + bitvec_t *word_active; /**< array of active flags for all words. */ + + /** + * Each node in the HMM tree structure may point to a set of words + * whose last phone would follow that node in the tree structure + * (but is not included in the tree structure for reasons + * explained above). The channel node points to one word in this + * set of words. The remaining words are linked through + * homophone_set[]. + * + * Single-phone words are not represented in the HMM tree; they + * are kept in word_chan. + * + * Specifically, homophone_set[w] = wid of next word in the same + * set as w. + */ + int32 *homophone_set; + int32 *single_phone_wid; /**< list of single-phone word ids */ + int32 n_1ph_words; /**< Number single phone words in dict (total) */ + int32 n_1ph_LMwords; /**< Number single phone dict words also in LM; + these come first in single_phone_wid */ + /** + * Array of active channels for current and next frame. + * + * In any frame, only some HMM tree nodes are active. + * active_chan_list[f mod 2] = list of nonroot channels in the HMM + * tree active in frame f. + */ + chan_t ***active_chan_list; + int32 n_active_chan[2]; /**< Number entries in active_chan_list */ + /** + * Array of active multi-phone words for current and next frame. + * + * Similarly to active_chan_list, active_word_list[f mod 2] = list + * of word ids for which active channels exist in word_chan in + * frame f. + * + * Statically allocated single-phone words are always active and + * should not appear in this list. + */ + int32 **active_word_list; + int32 n_active_word[2]; /**< Number entries in active_word_list */ + + /* + * FIXME: Document all of these bits. + */ + lastphn_cand_t *lastphn_cand; + int32 n_lastphn_cand; + last_ltrans_t *last_ltrans; /* one per word */ + int32 cand_sf_alloc; + cand_sf_t *cand_sf; + bestbp_rc_t *bestbp_rc; + + bptbl_t *bp_table; /* Forward pass lattice */ + int32 bpidx; /* First free BPTable entry */ + int32 bp_table_size; + int32 *bscore_stack; /* Score stack for all possible right contexts */ + int32 bss_head; /* First free BScoreStack entry */ + int32 bscore_stack_size; + + int32 n_frame_alloc; /**< Number of frames allocated in bp_table_idx and friends. */ + int32 n_frame; /**< Number of frames actually present. */ + int32 *bp_table_idx; /* First BPTable entry for each frame */ + int32 *word_lat_idx; /* BPTable index for any word in current frame; + cleared before each frame */ + + /* + * Flat lexicon (2nd pass) search stuff. + */ + ps_latnode_t **frm_wordlist; /**< List of active words in each frame. */ + int32 *fwdflat_wordlist; /**< List of active word IDs for utterance. */ + bitvec_t *expand_word_flag; + int32 *expand_word_list; + int32 n_expand_words; + int32 min_ef_width; + int32 max_sf_win; + float32 fwdflat_fwdtree_lw_ratio; + + int32 best_score; /**< Best Viterbi path score. */ + int32 last_phone_best_score; /**< Best Viterbi path score for last phone. */ + int32 renormalized; + + /* + * DAG (3rd pass) search stuff. + */ + float32 bestpath_fwdtree_lw_ratio; + float32 ascale; /**< Acoustic score scale for posterior probabilities. */ + + ngram_search_stats_t st; /**< Various statistics for profiling. */ + ptmr_t fwdtree_perf; + ptmr_t fwdflat_perf; + ptmr_t bestpath_perf; + int32 n_tot_frame; + + /* A collection of beam widths. */ + int32 beam; + int32 dynamic_beam; + int32 pbeam; + int32 wbeam; + int32 lpbeam; + int32 lponlybeam; + int32 fwdflatbeam; + int32 fwdflatwbeam; + int32 fillpen; + int32 silpen; + int32 wip; + int32 nwpen; + int32 pip; + int32 maxwpf; + int32 maxhmmpf; +}; +typedef struct ngram_search_s ngram_search_t; + +/** + * Initialize the N-Gram search module. + */ +ps_search_t *ngram_search_init(const char *name, + ngram_model_t *lm, + cmd_ln_t *config, + acmod_t *acmod, + dict_t *dict, + dict2pid_t *d2p); + +/** + * Finalize the N-Gram search module. + */ +void ngram_search_free(ps_search_t *ngs); + +/** + * Record the current frame's index in the backpointer table. + * + * @return the current backpointer index. + */ +int ngram_search_mark_bptable(ngram_search_t *ngs, int frame_idx); + +/** + * Enter a word in the backpointer table. + */ +void ngram_search_save_bp(ngram_search_t *ngs, int frame_idx, int32 w, + int32 score, int32 path, int32 rc); + +/** + * Allocate last phone channels for all possible right contexts for word w. + */ +void ngram_search_alloc_all_rc(ngram_search_t *ngs, int32 w); + +/** + * Allocate last phone channels for all possible right contexts for word w. + */ +void ngram_search_free_all_rc(ngram_search_t *ngs, int32 w); + +/** + * Find the best word exit for the current frame in the backpointer table. + * + * @return the backpointer index of the best word exit. + */ +int ngram_search_find_exit(ngram_search_t *ngs, int frame_idx, int32 *out_best_score); + +/** + * Backtrace from a given backpointer index to obtain a word hypothesis. + * + * @return a read-only string with the best hypothesis. + */ +char const *ngram_search_bp_hyp(ngram_search_t *ngs, int bpidx); + +/** + * Compute language and acoustic scores for backpointer table entries. + */ +void ngram_compute_seg_scores(ngram_search_t *ngs, float32 lwf); + +/** + * Construct a word lattice from the current hypothesis. + */ +ps_lattice_t *ngram_search_lattice(ps_search_t *search); + +/** + * Get the exit score for a backpointer entry with a given right context. + */ +int32 ngram_search_exit_score(ngram_search_t *ngs, bptbl_t *pbe, int rcphone); + +/** + * Sets the global language model. + * + * Sets the language model to use if nothing was passed in configuration + */ +void ngram_search_set_lm(ngram_model_t *lm); + +#endif /* __NGRAM_SEARCH_H__ */ diff --git a/include/ngram_search_fwdflat.h b/include/ngram_search_fwdflat.h new file mode 100644 index 0000000..026b397 --- /dev/null +++ b/include/ngram_search_fwdflat.h @@ -0,0 +1,81 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 2008 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +/** + * @file ngram_search_fwdflat.h Flat lexicon based Viterbi search. + */ + +#ifndef __NGRAM_SEARCH_FWDFLAT_H__ +#define __NGRAM_SEARCH_FWDFLAT_H__ + +/* SphinxBase headers. */ + +/* Local headers. */ +#include "ngram_search.h" + +/** + * Initialize N-Gram search for fwdflat decoding. + */ +void ngram_fwdflat_init(ngram_search_t *ngs); + +/** + * Release memory associated with fwdflat decoding. + */ +void ngram_fwdflat_deinit(ngram_search_t *ngs); + +/** + * Rebuild search structures for updated language models. + */ +int ngram_fwdflat_reinit(ngram_search_t *ngs); + +/** + * Start fwdflat decoding for an utterance. + */ +void ngram_fwdflat_start(ngram_search_t *ngs); + +/** + * Search one frame forward in an utterance. + */ +int ngram_fwdflat_search(ngram_search_t *ngs, int frame_idx); + +/** + * Finish fwdflat decoding for an utterance. + */ +void ngram_fwdflat_finish(ngram_search_t *ngs); + + +#endif /* __NGRAM_SEARCH_FWDFLAT_H__ */ diff --git a/include/ngram_search_fwdtree.h b/include/ngram_search_fwdtree.h new file mode 100644 index 0000000..8063ab7 --- /dev/null +++ b/include/ngram_search_fwdtree.h @@ -0,0 +1,83 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 2008 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +/** + * @file ngram_search_fwdtree.h Lexicon tree based Viterbi search. + */ + +#ifndef __NGRAM_SEARCH_FWDTREE_H__ +#define __NGRAM_SEARCH_FWDTREE_H__ + +/* SphinxBase headers. */ + +/* Local headers. */ +#include "ngram_search.h" + +/** + * Initialize N-Gram search for fwdtree decoding. + */ +void ngram_fwdtree_init(ngram_search_t *ngs); + +/** + * Release memory associated with fwdtree decoding. + */ +void ngram_fwdtree_deinit(ngram_search_t *ngs); + +/** + * Rebuild search structures for updated language models. + */ +int ngram_fwdtree_reinit(ngram_search_t *ngs); + +/** + * Start fwdtree decoding for an utterance. + */ +void ngram_fwdtree_start(ngram_search_t *ngs); + +/** + * Search one frame forward in an utterance. + * + * @return Number of frames searched (either 0 or 1). + */ +int ngram_fwdtree_search(ngram_search_t *ngs, int frame_idx); + +/** + * Finish fwdtree decoding for an utterance. + */ +void ngram_fwdtree_finish(ngram_search_t *ngs); + + +#endif /* __NGRAM_SEARCH_FWDTREE_H__ */ diff --git a/include/phone_loop_search.h b/include/phone_loop_search.h new file mode 100644 index 0000000..db776e9 --- /dev/null +++ b/include/phone_loop_search.h @@ -0,0 +1,102 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 2008 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +/** + * @file phone_loop_search.h Fast and rough context-independent + * phoneme loop search. + * + * This exists for the purposes of phoneme lookahead, and thus it + * actually does not do phoneme recognition (it wouldn't be very + * accurate anyway). + */ + +#ifndef __PHONE_LOOP_SEARCH_H__ +#define __PHONE_LOOP_SEARCH_H__ + +/* SphinxBase headers. */ +#include +#include +#include +#include + +/* Local headers. */ +#include "pocketsphinx_internal.h" +#include "hmm.h" + +/** + * Renormalization event. + */ +struct phone_loop_renorm_s { + int frame_idx; /**< Frame of renormalization. */ + int32 norm; /**< Normalization constant. */ +}; +typedef struct phone_loop_renorm_s phone_loop_renorm_t; + +/** + * Phone loop search structure. + */ +struct phone_loop_search_s { + ps_search_t base; /**< Base search structure. */ + hmm_t *hmms; /**< Basic HMM structures for CI phones. */ + hmm_context_t *hmmctx; /**< HMM context structure. */ + int16 frame; /**< Current frame being searched. */ + int16 n_phones; /**< Size of phone array. */ + int32 **pen_buf; /**< Penalty buffer */ + int16 pen_buf_ptr; /**< Pointer for frame to fill in penalty buffer */ + int32 *penalties; /**< Penalties for CI phones in current frame */ + float64 penalty_weight; /**< Weighting factor for penalties */ + + int32 best_score; /**< Best Viterbi score in current frame. */ + int32 beam; /**< HMM pruning beam width. */ + int32 pbeam; /**< Phone exit pruning beam width. */ + int32 pip; /**< Phone insertion penalty ("language score"). */ + int window; /**< Window size for phoneme lookahead */ + glist_t renorm; /**< List of renormalizations. */ +}; +typedef struct phone_loop_search_s phone_loop_search_t; + +ps_search_t *phone_loop_search_init(cmd_ln_t *config, + acmod_t *acmod, + dict_t *dict); + +/** + * Return lookahead heuristic score for a specific phone. + */ +#define phone_loop_search_score(pls,ci) \ + ((pls == NULL) ? 0 : (pls->penalties[ci])) + +#endif /* __PHONE_LOOP_SEARCH_H__ */ diff --git a/include/pocketsphinx_internal.h b/include/pocketsphinx_internal.h new file mode 100644 index 0000000..3f7dd98 --- /dev/null +++ b/include/pocketsphinx_internal.h @@ -0,0 +1,234 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 2008 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +/** + * @file pocketsphinx_internal.h Internal implementation of + * PocketSphinx decoder. + * @author David Huggins-Daines + */ + +#ifndef __POCKETSPHINX_INTERNAL_H__ +#define __POCKETSPHINX_INTERNAL_H__ + +/* SphinxBase headers. */ +#include +#include +#include +#include +#include +#include + +/* Local headers. */ +#include "pocketsphinx.h" +#include "acmod.h" +#include "dict.h" +#include "dict2pid.h" + +/** + * Search algorithm structure. + */ +typedef struct ps_search_s ps_search_t; + + +/* Search names*/ +#define PS_DEFAULT_SEARCH "_default" +#define PS_DEFAULT_PL_SEARCH "_default_pl" + +/* Search types */ +#define PS_SEARCH_TYPE_KWS "kws" +#define PS_SEARCH_TYPE_FSG "fsg" +#define PS_SEARCH_TYPE_NGRAM "ngram" +#define PS_SEARCH_TYPE_ALLPHONE "allphone" +#define PS_SEARCH_TYPE_STATE_ALIGN "state_align" +#define PS_SEARCH_TYPE_PHONE_LOOP "phone_loop" + +/** + * V-table for search algorithm. + */ +typedef struct ps_searchfuncs_s { + int (*start)(ps_search_t *search); + int (*step)(ps_search_t *search, int frame_idx); + int (*finish)(ps_search_t *search); + int (*reinit)(ps_search_t *search, dict_t *dict, dict2pid_t *d2p); + void (*free)(ps_search_t *search); + + ps_lattice_t *(*lattice)(ps_search_t *search); + char const *(*hyp)(ps_search_t *search, int32 *out_score); + int32 (*prob)(ps_search_t *search); + ps_seg_t *(*seg_iter)(ps_search_t *search); +} ps_searchfuncs_t; + +/** + * Base structure for search module. + */ +struct ps_search_s { + ps_searchfuncs_t *vt; /**< V-table of search methods. */ + + char *type; + char *name; + + ps_search_t *pls; /**< Phoneme loop for lookahead. */ + cmd_ln_t *config; /**< Configuration. */ + acmod_t *acmod; /**< Acoustic model. */ + dict_t *dict; /**< Pronunciation dictionary. */ + dict2pid_t *d2p; /**< Dictionary to senone mappings. */ + char *hyp_str; /**< Current hypothesis string. */ + ps_lattice_t *dag; /**< Current hypothesis word graph. */ + ps_latlink_t *last_link; /**< Final link in best path. */ + int32 post; /**< Utterance posterior probability. */ + int32 n_words; /**< Number of words known to search (may + be less than in the dictionary) */ + + /* Magical word IDs that must exist in the dictionary: */ + int32 start_wid; /**< Start word ID. */ + int32 silence_wid; /**< Silence word ID. */ + int32 finish_wid; /**< Finish word ID. */ +}; + +#define ps_search_base(s) ((ps_search_t *)s) +#define ps_search_config(s) ps_search_base(s)->config +#define ps_search_acmod(s) ps_search_base(s)->acmod +#define ps_search_dict(s) ps_search_base(s)->dict +#define ps_search_dict2pid(s) ps_search_base(s)->d2p +#define ps_search_dag(s) ps_search_base(s)->dag +#define ps_search_last_link(s) ps_search_base(s)->last_link +#define ps_search_post(s) ps_search_base(s)->post +#define ps_search_lookahead(s) ps_search_base(s)->pls +#define ps_search_n_words(s) ps_search_base(s)->n_words + +#define ps_search_type(s) ps_search_base(s)->type +#define ps_search_name(s) ps_search_base(s)->name +#define ps_search_start(s) (*(ps_search_base(s)->vt->start))(s) +#define ps_search_step(s,i) (*(ps_search_base(s)->vt->step))(s,i) +#define ps_search_finish(s) (*(ps_search_base(s)->vt->finish))(s) +#define ps_search_reinit(s,d,d2p) (*(ps_search_base(s)->vt->reinit))(s,d,d2p) +#define ps_search_free(s) (*(ps_search_base(s)->vt->free))(s) +#define ps_search_lattice(s) (*(ps_search_base(s)->vt->lattice))(s) +#define ps_search_hyp(s,sc) (*(ps_search_base(s)->vt->hyp))(s,sc) +#define ps_search_prob(s) (*(ps_search_base(s)->vt->prob))(s) +#define ps_search_seg_iter(s) (*(ps_search_base(s)->vt->seg_iter))(s) + +/* For convenience... */ +#define ps_search_silence_wid(s) ps_search_base(s)->silence_wid +#define ps_search_start_wid(s) ps_search_base(s)->start_wid +#define ps_search_finish_wid(s) ps_search_base(s)->finish_wid + +/** + * Initialize base structure. + */ +void ps_search_init(ps_search_t *search, ps_searchfuncs_t *vt, + const char *type, const char *name, + cmd_ln_t *config, acmod_t *acmod, dict_t *dict, + dict2pid_t *d2p); + + +/** + * Free search + */ +void ps_search_base_free(ps_search_t *search); + +/** + * Re-initialize base structure with new dictionary. + */ +void ps_search_base_reinit(ps_search_t *search, dict_t *dict, + dict2pid_t *d2p); + +typedef struct ps_segfuncs_s { + ps_seg_t *(*seg_next)(ps_seg_t *seg); + void (*seg_free)(ps_seg_t *seg); +} ps_segfuncs_t; + +/** + * Base structure for hypothesis segmentation iterator. + */ +struct ps_seg_s { + ps_segfuncs_t *vt; /**< V-table of seg methods */ + ps_search_t *search; /**< Search object from whence this came */ + char const *word; /**< Word string (pointer into dictionary hash) */ + frame_idx_t sf; /**< Start frame. */ + frame_idx_t ef; /**< End frame. */ + int32 ascr; /**< Acoustic score. */ + int32 lscr; /**< Language model score. */ + int32 prob; /**< Log posterior probability. */ + /* This doesn't need to be 32 bits, so once the scores above are + * reduced to 16 bits (or less!), this will be too. */ + int32 lback; /**< Language model backoff. */ + /* Not sure if this should be here at all. */ + float32 lwf; /**< Language weight factor (for second-pass searches) */ +}; + +#define ps_search_seg_next(seg) (*(seg->vt->seg_next))(seg) +#define ps_search_seg_free(s) (*(seg->vt->seg_free))(seg) + + +/** + * Decoder object. + */ +struct ps_decoder_s { + /* Model parameters and such. */ + cmd_ln_t *config; /**< Configuration. */ + int refcount; /**< Reference count. */ + + /* Basic units of computation. */ + acmod_t *acmod; /**< Acoustic model. */ + dict_t *dict; /**< Pronunciation dictionary. */ + dict2pid_t *d2p; /**< Dictionary to senone mapping. */ + logmath_t *lmath; /**< Log math computation. */ + + /* Search modules. */ + hash_table_t *searches; /**< Set of search modules. */ + /* TODO: Convert this to a stack of searches each with their own + * lookahead value. */ + ps_search_t *search; /**< Currently active search module. */ + ps_search_t *phone_loop; /**< Phone loop search for lookahead. */ + int pl_window; /**< Window size for phoneme lookahead. */ + + /* Utterance-processing related stuff. */ + uint32 uttno; /**< Utterance counter. */ + ptmr_t perf; /**< Performance counter for all of decoding. */ + uint32 n_frame; /**< Total number of frames processed. */ + char const *mfclogdir; /**< Log directory for MFCC files. */ + char const *rawlogdir; /**< Log directory for audio files. */ + char const *senlogdir; /**< Log directory for senone score files. */ +}; + + +struct ps_search_iter_s { + hash_iter_t itor; +}; + +#endif /* __POCKETSPHINX_INTERNAL_H__ */ diff --git a/include/ps_alignment.h b/include/ps_alignment.h new file mode 100644 index 0000000..4774bef --- /dev/null +++ b/include/ps_alignment.h @@ -0,0 +1,190 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 2010 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +/** + * @file ps_alignment.h Multi-level alignment structure + */ + +#ifndef __PS_ALIGNMENT_H__ +#define __PS_ALIGNMENT_H__ + +/* System headers. */ + +/* SphinxBase headers. */ +#include + +/* Local headers. */ +#include "dict2pid.h" +#include "hmm.h" + +#define PS_ALIGNMENT_NONE ((uint16)0xffff) + +struct ps_alignment_entry_s { + union { + int32 wid; + struct { + uint16 ssid; + uint16 cipid; + uint16 tmatid; + } pid; + uint16 senid; + } id; + int16 start; + int16 duration; + int32 score; + uint16 parent; + uint16 child; +}; +typedef struct ps_alignment_entry_s ps_alignment_entry_t; + +struct ps_alignment_vector_s { + ps_alignment_entry_t *seq; + uint16 n_ent, n_alloc; +}; +typedef struct ps_alignment_vector_s ps_alignment_vector_t; + +struct ps_alignment_s { + dict2pid_t *d2p; + ps_alignment_vector_t word; + ps_alignment_vector_t sseq; + ps_alignment_vector_t state; +}; +typedef struct ps_alignment_s ps_alignment_t; + +struct ps_alignment_iter_s { + ps_alignment_t *al; + ps_alignment_vector_t *vec; + int pos; +}; +typedef struct ps_alignment_iter_s ps_alignment_iter_t; + +/** + * Create a new, empty alignment. + */ +ps_alignment_t *ps_alignment_init(dict2pid_t *d2p); + +/** + * Release an alignment + */ +int ps_alignment_free(ps_alignment_t *al); + +/** + * Append a word. + */ +int ps_alignment_add_word(ps_alignment_t *al, + int32 wid, int duration); + +/** + * Populate lower layers using available word information. + */ +int ps_alignment_populate(ps_alignment_t *al); + +/** + * Populate lower layers using context-independent phones. + */ +int ps_alignment_populate_ci(ps_alignment_t *al); + +/** + * Propagate timing information up from state sequence. + */ +int ps_alignment_propagate(ps_alignment_t *al); + +/** + * Number of words. + */ +int ps_alignment_n_words(ps_alignment_t *al); + +/** + * Number of phones. + */ +int ps_alignment_n_phones(ps_alignment_t *al); + +/** + * Number of states. + */ +int ps_alignment_n_states(ps_alignment_t *al); + +/** + * Iterate over the alignment starting at the first word. + */ +ps_alignment_iter_t *ps_alignment_words(ps_alignment_t *al); + +/** + * Iterate over the alignment starting at the first phone. + */ +ps_alignment_iter_t *ps_alignment_phones(ps_alignment_t *al); + +/** + * Iterate over the alignment starting at the first state. + */ +ps_alignment_iter_t *ps_alignment_states(ps_alignment_t *al); + +/** + * Get the alignment entry pointed to by an iterator. + */ +ps_alignment_entry_t *ps_alignment_iter_get(ps_alignment_iter_t *itor); + +/** + * Move alignment iterator to given index. + */ +ps_alignment_iter_t *ps_alignment_iter_goto(ps_alignment_iter_t *itor, int pos); + +/** + * Move an alignment iterator forward. + */ +ps_alignment_iter_t *ps_alignment_iter_next(ps_alignment_iter_t *itor); + +/** + * Move an alignment iterator back. + */ +ps_alignment_iter_t *ps_alignment_iter_prev(ps_alignment_iter_t *itor); + +/** + * Get a new iterator starting at the parent of the current node. + */ +ps_alignment_iter_t *ps_alignment_iter_up(ps_alignment_iter_t *itor); +/** + * Get a new iterator starting at the first child of the current node. + */ +ps_alignment_iter_t *ps_alignment_iter_down(ps_alignment_iter_t *itor); + +/** + * Release an iterator before completing all iterations. + */ +int ps_alignment_iter_free(ps_alignment_iter_t *itor); + +#endif /* __PS_ALIGNMENT_H__ */ diff --git a/include/ps_lattice_internal.h b/include/ps_lattice_internal.h new file mode 100644 index 0000000..4e5f7dd --- /dev/null +++ b/include/ps_lattice_internal.h @@ -0,0 +1,282 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 2008 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +/** + * @file ps_lattice_internal.h Word graph search implementation + */ + +#ifndef __PS_LATTICE_INTERNAL_H__ +#define __PS_LATTICE_INTERNAL_H__ + +/** + * Linked list of DAG link pointers. + * + * Because the same link structure is used for forward and reverse + * links, as well as for the agenda used in bestpath search, we can't + * store the list pointer inside latlink_t. We could use glist_t + * here, but it wastes 4 bytes per entry on 32-bit machines. + */ +typedef struct latlink_list_s { + ps_latlink_t *link; + struct latlink_list_s *next; +} latlink_list_t; + +/** + * Word graph structure used in bestpath/nbest search. + */ +struct ps_lattice_s { + int refcount; /**< Reference count. */ + + logmath_t *lmath; /**< Log-math object. */ + ps_search_t *search; /**< Search (if generated by search). */ + dict_t *dict; /**< Dictionary for this DAG. */ + int32 silence; /**< Silence word ID. */ + int32 frate; /**< Frame rate. */ + + ps_latnode_t *nodes; /**< List of all nodes. */ + ps_latnode_t *start; /**< Starting node. */ + ps_latnode_t *end; /**< Ending node. */ + + frame_idx_t n_frames; /**< Number of frames for this utterance. */ + int32 n_nodes; /**< Number of nodes in this lattice. */ + int32 final_node_ascr; /**< Acoustic score of implicit link exiting final node. */ + int32 norm; /**< Normalizer for posterior probabilities. */ + char *hyp_str; /**< Current hypothesis string. */ + + listelem_alloc_t *latnode_alloc; /**< Node allocator for this DAG. */ + listelem_alloc_t *latlink_alloc; /**< Link allocator for this DAG. */ + listelem_alloc_t *latlink_list_alloc; /**< List element allocator for this DAG. */ + + /* This will probably be replaced with a heap. */ + latlink_list_t *q_head; /**< Queue of links for traversal. */ + latlink_list_t *q_tail; /**< Queue of links for traversal. */ +}; + +/** + * Links between DAG nodes. + * + * A link corresponds to a single hypothesized instance of a word with + * a given start and end point. + + */ +struct ps_latlink_s { + struct ps_latnode_s *from; /**< From node */ + struct ps_latnode_s *to; /**< To node */ + struct ps_latlink_s *best_prev; + int32 ascr; /**< Score for from->wid (from->sf to this->ef) */ + int32 path_scr; /**< Best path score from root of DAG */ + frame_idx_t ef; /**< Ending frame of this word */ + int32 alpha; /**< Forward probability of this link P(w,o_1^{ef}) */ + int32 beta; /**< Backward probability of this link P(w|o_{ef+1}^T) */ +}; + +/** + * DAG nodes. + * + * A node corresponds to a number of hypothesized instances of a word + * which all share the same starting point. + */ +struct ps_latnode_s { + int32 id; /**< Unique id for this node */ + int32 wid; /**< Dictionary word id */ + int32 basewid; /**< Dictionary base word id */ + /* FIXME: These are (ab)used to store backpointer indices, therefore they MUST be 32 bits. */ + int32 fef; /**< First end frame */ + int32 lef; /**< Last end frame */ + frame_idx_t sf; /**< Start frame */ + int16 reachable; /**< From \verbatim \endverbatim or \verbatim \endverbatim */ + int32 node_id; /**< Node from fsg model, used to map lattice back to model */ + union { + glist_t velist; /**< List of history entries with different lmstate (tst only) */ + int32 fanin; /**< Number nodes with links to this node */ + int32 rem_score; /**< Estimated best score from node.sf to end */ + int32 best_exit; /**< Best exit score (used for final nodes only) */ + } info; + latlink_list_t *exits; /**< Links out of this node */ + latlink_list_t *entries; /**< Links into this node */ + + struct ps_latnode_s *alt; /**< Node with alternate pronunciation for this word */ + struct ps_latnode_s *next; /**< Next node in DAG (no ordering implied) */ +}; + +/** + * Segmentation "iterator" for backpointer table results. + */ +typedef struct dag_seg_s { + ps_seg_t base; /**< Base structure. */ + ps_latlink_t **links; /**< Array of lattice links. */ + int32 norm; /**< Normalizer for posterior probabilities. */ + int16 n_links; /**< Number of lattice links. */ + int16 cur; /**< Current position in bpidx. */ +} dag_seg_t; + +/** + * Partial path structure used in N-best (A*) search. + * + * Each partial path (latpath_t) is constructed by extending another + * partial path--parent--by one node. + */ +typedef struct ps_latpath_s { + ps_latnode_t *node; /**< Node ending this path. */ + struct ps_latpath_s *parent; /**< Previous element in this path. */ + struct ps_latpath_s *next; /**< Pointer to next path in list of paths. */ + int32 score; /**< Exact score from start node up to node->sf. */ +} ps_latpath_t; + +/** + * A* search structure. + */ +typedef struct ps_astar_s { + ps_lattice_t *dag; + ngram_model_t *lmset; + float32 lwf; + + frame_idx_t sf; + frame_idx_t ef; + int32 w1; + int32 w2; + + int32 n_hyp_tried; + int32 n_hyp_insert; + int32 n_hyp_reject; + int32 insert_depth; + int32 n_path; + + ps_latpath_t *path_list; + ps_latpath_t *path_tail; + ps_latpath_t *top; + + glist_t hyps; /**< List of hypothesis strings. */ + listelem_alloc_t *latpath_alloc; /**< Path allocator for N-best search. */ +} ps_astar_t; + +/** + * Segmentation "iterator" for A* search results. + */ +typedef struct astar_seg_s { + ps_seg_t base; + ps_latnode_t **nodes; + int n_nodes; + int cur; +} astar_seg_t; + +/** + * Construct an empty word graph with reference to a search structure. + */ +ps_lattice_t *ps_lattice_init_search(ps_search_t *search, int n_frame); + +/** + * Insert penalty for fillers + */ +void ps_lattice_penalize_fillers(ps_lattice_t *dag, int32 silpen, int32 fillpen); + +/** + * Remove nodes marked as unreachable. + */ +void ps_lattice_delete_unreachable(ps_lattice_t *dag); + +/** + * Add an edge to the traversal queue. + */ +void ps_lattice_pushq(ps_lattice_t *dag, ps_latlink_t *link); + +/** + * Remove an edge from the traversal queue. + */ +ps_latlink_t *ps_lattice_popq(ps_lattice_t *dag); + +/** + * Clear and reset the traversal queue. + */ +void ps_lattice_delq(ps_lattice_t *dag); + +/** + * Create a new lattice link element. + */ +latlink_list_t *latlink_list_new(ps_lattice_t *dag, ps_latlink_t *link, + latlink_list_t *next); + +/** + * Get hypothesis string after bestpath search. + */ +char const *ps_lattice_hyp(ps_lattice_t *dag, ps_latlink_t *link); + +/** + * Get hypothesis segmentation iterator after bestpath search. + */ +ps_seg_t *ps_lattice_seg_iter(ps_lattice_t *dag, ps_latlink_t *link, + float32 lwf); + +/** + * Begin N-Gram based A* search on a word graph. + * + * @param sf Starting frame for N-best search. + * @param ef Ending frame for N-best search, or -1 for last frame. + * @param w1 First context word, or -1 for none. + * @param w2 Second context word, or -1 for none. + * @return 0 for success, <0 on error. + */ +ps_astar_t *ps_astar_start(ps_lattice_t *dag, + ngram_model_t *lmset, + float32 lwf, + int sf, int ef, + int w1, int w2); + +/** + * Find next best hypothesis of A* on a word graph. + * + * @return a complete path, or NULL if no more hypotheses exist. + */ +ps_latpath_t *ps_astar_next(ps_astar_t *nbest); + +/** + * Finish N-best search, releasing resources associated with it. + */ +void ps_astar_finish(ps_astar_t *nbest); + +/** + * Get hypothesis string from A* search. + */ +char const *ps_astar_hyp(ps_astar_t *nbest, ps_latpath_t *path); + +/** + * Get hypothesis segmentation from A* search. + */ +ps_seg_t *ps_astar_seg_iter(ps_astar_t *astar, ps_latpath_t *path, float32 lwf); + + +#endif /* __PS_LATTICE_INTERNAL_H__ */ diff --git a/include/ptm_mgau.h b/include/ptm_mgau.h new file mode 100644 index 0000000..0b3ac63 --- /dev/null +++ b/include/ptm_mgau.h @@ -0,0 +1,103 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 1999-2010 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ +/** + * @file ptm_mgau.h Fast phonetically-tied mixture evaluation. + * @author David Huggins-Daines + */ + +#ifndef __PTM_MGAU_H__ +#define __PTM_MGAU_H__ + +/* SphinxBase headesr. */ +#include +#include +#include + +/* Local headers. */ +#include "acmod.h" +#include "hmm.h" +#include "bin_mdef.h" +#include "ms_gauden.h" + +typedef struct ptm_mgau_s ptm_mgau_t; + +typedef struct ptm_topn_s { + int32 cw; /**< Codeword index. */ + int32 score; /**< Score. */ +} ptm_topn_t; + +typedef struct ptm_fast_eval_s { + ptm_topn_t ***topn; /**< Top-N for each codebook (mgau x feature x topn) */ + bitvec_t *mgau_active; /**< Set of active codebooks */ +} ptm_fast_eval_t; + +struct ptm_mgau_s { + ps_mgau_t base; /**< base structure. */ + cmd_ln_t *config; /**< Configuration parameters */ + gauden_t *g; /**< Set of Gaussians. */ + int32 n_sen; /**< Number of senones. */ + uint8 *sen2cb; /**< Senone to codebook mapping. */ + uint8 ***mixw; /**< Mixture weight distributions by feature, codeword, senone */ + mmio_file_t *sendump_mmap;/* Memory map for mixw (or NULL if not mmap) */ + uint8 *mixw_cb; /* Mixture weight codebook, if any (assume it contains 16 values) */ + int16 max_topn; + int16 ds_ratio; + + ptm_fast_eval_t *hist; /**< Fast evaluation info for past frames. */ + ptm_fast_eval_t *f; /**< Fast eval info for current frame. */ + int n_fast_hist; /**< Number of past frames tracked. */ + + /* Log-add table for compressed values. */ + logmath_t *lmath_8b; + /* Log-add object for reloading means/variances. */ + logmath_t *lmath; +}; + +ps_mgau_t *ptm_mgau_init(acmod_t *acmod, bin_mdef_t *mdef); +void ptm_mgau_free(ps_mgau_t *s); +int ptm_mgau_frame_eval(ps_mgau_t *s, + int16 *senone_scores, + uint8 *senone_active, + int32 n_senone_active, + mfcc_t **featbuf, + int32 frame, + int32 compallsen); +int ptm_mgau_mllr_transform(ps_mgau_t *s, + ps_mllr_t *mllr); + + +#endif /* __PTM_MGAU_H__ */ diff --git a/include/s2_semi_mgau.h b/include/s2_semi_mgau.h new file mode 100644 index 0000000..f127b5d --- /dev/null +++ b/include/s2_semi_mgau.h @@ -0,0 +1,98 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 1999-2004 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ +/* + * Interface for "semi-continuous vector quantization", a.k.a. Sphinx2 + * fast GMM computation. + */ + +#ifndef __S2_SEMI_MGAU_H__ +#define __S2_SEMI_MGAU_H__ + +/* SphinxBase headesr. */ +#include +#include +#include + +/* Local headers. */ +#include "acmod.h" +#include "hmm.h" +#include "bin_mdef.h" +#include "ms_gauden.h" + +typedef struct vqFeature_s vqFeature_t; + +typedef struct s2_semi_mgau_s s2_semi_mgau_t; +struct s2_semi_mgau_s { + ps_mgau_t base; /**< base structure. */ + cmd_ln_t *config; /* configuration parameters */ + + gauden_t *g; /* Set of Gaussians (pointers below point in here and will go away soon) */ + + uint8 ***mixw; /* mixture weight distributions */ + mmio_file_t *sendump_mmap;/* memory map for mixw (or NULL if not mmap) */ + + uint8 *mixw_cb; /* mixture weight codebook, if any (assume it contains 16 values) */ + int32 n_sen; /* Number of senones */ + uint8 *topn_beam; /* Beam for determining per-frame top-N densities */ + int16 max_topn; + int16 ds_ratio; + + vqFeature_t ***topn_hist; /**< Top-N scores and codewords for past frames. */ + uint8 **topn_hist_n; /**< Variable top-N for past frames. */ + vqFeature_t **f; /**< Topn-N for currently scoring frame. */ + int n_topn_hist; /**< Number of past frames tracked. */ + + /* Log-add table for compressed values. */ + logmath_t *lmath_8b; + /* Log-add object for reloading means/variances. */ + logmath_t *lmath; +}; + +ps_mgau_t *s2_semi_mgau_init(acmod_t *acmod); +void s2_semi_mgau_free(ps_mgau_t *s); +int s2_semi_mgau_frame_eval(ps_mgau_t *s, + int16 *senone_scores, + uint8 *senone_active, + int32 n_senone_active, + mfcc_t **featbuf, + int32 frame, + int32 compallsen); +int s2_semi_mgau_mllr_transform(ps_mgau_t *s, + ps_mllr_t *mllr); + + +#endif /* __S2_SEMI_MGAU_H__ */ diff --git a/include/s3types.h b/include/s3types.h new file mode 100644 index 0000000..b40f4c1 --- /dev/null +++ b/include/s3types.h @@ -0,0 +1,99 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 1999-2004 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +#ifndef _S3_S3TYPES_H_ +#define _S3_S3TYPES_H_ + +#include +#include + +#include +#include +#include + +/** \file s3types.h + * \brief Size definition of semantically units. Common for both s3 and s3.X decoder. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Size definitions for more semantially meaningful units. + * Illegal value definitions, limits, and tests for specific types. + * NOTE: Types will be either int32 or smaller; only smaller ones may be unsigned (i.e., + * no type will be uint32). + */ + +typedef int16 s3cipid_t; /** Ci phone id */ +#define BAD_S3CIPID ((s3cipid_t) -1) +#define NOT_S3CIPID(p) ((p)<0) +#define IS_S3CIPID(p) ((p)>=0) +#define MAX_S3CIPID 32767 + +/*#define MAX_S3CIPID 127*/ + +typedef int32 s3pid_t; /** Phone id (triphone or ciphone) */ +#define BAD_S3PID ((s3pid_t) -1) +#define NOT_S3PID(p) ((p)<0) +#define IS_S3PID(p) ((p)>=0) +#define MAX_S3PID ((int32)0x7ffffffe) + +typedef uint16 s3ssid_t; /** Senone sequence id (triphone or ciphone) */ +#define BAD_S3SSID ((s3ssid_t) 0xffff) +#define NOT_S3SSID(p) ((p) == BAD_S3SSID) +#define IS_S3SSID(p) ((p) != BAD_S3SSID) +#define MAX_S3SSID ((s3ssid_t)0xfffe) + +typedef int32 s3tmatid_t; /** Transition matrix id; there can be as many as pids */ +#define BAD_S3TMATID ((s3tmatid_t) -1) +#define NOT_S3TMATID(t) ((t)<0) +#define IS_S3TMATID(t) ((t)>=0) +#define MAX_S3TMATID ((int32)0x7ffffffe) + +typedef int32 s3wid_t; /** Dictionary word id */ +#define BAD_S3WID ((s3wid_t) -1) +#define NOT_S3WID(w) ((w)<0) +#define IS_S3WID(w) ((w)>=0) +#define MAX_S3WID ((int32)0x7ffffffe) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/state_align_search.h b/include/state_align_search.h new file mode 100644 index 0000000..af102a6 --- /dev/null +++ b/include/state_align_search.h @@ -0,0 +1,87 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 2010 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +/** + * @file state_align_search.h State (and phone and word) alignment search. + */ + +#ifndef __STATE_ALIGN_SEARCH_H__ +#define __STATE_ALIGN_SEARCH_H__ + +/* SphinxBase headers. */ +#include + +/* Local headers. */ +#include "pocketsphinx_internal.h" +#include "ps_alignment.h" +#include "hmm.h" + + +/** + * History structure + */ +struct state_align_hist_s { + uint16 id; + int32 score; +}; +typedef struct state_align_hist_s state_align_hist_t; + +/** + * Phone loop search structure. + */ +struct state_align_search_s { + ps_search_t base; /**< Base search structure. */ + hmm_context_t *hmmctx; /**< HMM context structure. */ + ps_alignment_t *al; /**< Alignment structure being operated on. */ + hmm_t *hmms; /**< Vector of HMMs corresponding to phone level. */ + int n_phones; /**< Number of HMMs (phones). */ + + int frame; /**< Current frame being processed. */ + int32 best_score; /**< Best score in current frame. */ + + int n_emit_state; /**< Number of emitting states (tokens per frame) */ + state_align_hist_t *tokens; /**< Tokens (backpointers) for state alignment. */ + int n_fr_alloc; /**< Number of frames of tokens allocated. */ +}; +typedef struct state_align_search_s state_align_search_t; + +ps_search_t *state_align_search_init(const char *name, + cmd_ln_t *config, + acmod_t *acmod, + ps_alignment_t *al); + +#endif /* __STATE_ALIGN_SEARCH_H__ */ diff --git a/include/tied_mgau_common.h b/include/tied_mgau_common.h new file mode 100644 index 0000000..c8c5320 --- /dev/null +++ b/include/tied_mgau_common.h @@ -0,0 +1,121 @@ +/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* ==================================================================== + * Copyright (c) 1999-2010 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +/** + * @file tied_mgau_common.h + * @brief Common code shared between SC and PTM (tied-state) models. + */ + +#ifndef __TIED_MGAU_COMMON_H__ +#define __TIED_MGAU_COMMON_H__ + +#include +#include + +#define MGAU_MIXW_VERSION "1.0" /* Sphinx-3 file format version for mixw */ +#define MGAU_PARAM_VERSION "1.0" /* Sphinx-3 file format version for mean/var */ +#define NONE -1 +#define WORST_DIST (int32)(0x80000000) + +/** Subtract GMM component b (assumed to be positive) and saturate */ +#ifdef FIXED_POINT +#define GMMSUB(a,b) \ + (((a)-(b) > a) ? (INT_MIN) : ((a)-(b))) +/** Add GMM component b (assumed to be positive) and saturate */ +#define GMMADD(a,b) \ + (((a)+(b) < a) ? (INT_MAX) : ((a)+(b))) +#else +#define GMMSUB(a,b) ((a)-(b)) +#define GMMADD(a,b) ((a)+(b)) +#endif + +#ifndef MIN +#define MIN(a,b) ((a) < (b) ? (a) : (b)) +#endif + + +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) +#define LOGMATH_INLINE static inline +#elif defined(_MSC_VER) +#define LOGMATH_INLINE __inline +#else +#define LOGMATH_INLINE static +#endif + +/* Allocate 0..159 for negated quantized mixture weights and 0..96 for + * negated normalized acoustic scores, so that the combination of the + * two (for a single mixture) can never exceed 255. */ +#define MAX_NEG_MIXW 159 /**< Maximum negated mixture weight value. */ +#define MAX_NEG_ASCR 96 /**< Maximum negated acoustic score value. */ + +/** + * Quickly log-add two negated log probabilities. + * + * @param lmath The log-math object + * @param mlx A negative log probability (0 < mlx < 255) + * @param mly A negative log probability (0 < mly < 255) + * @return -log(exp(-mlx)+exp(-mly)) + * + * We can do some extra-fast log addition since we know that + * mixw+ascr is always less than 256 and hence x-y is also always less + * than 256. This relies on some cooperation from logmath_t which + * will never produce a logmath table smaller than 256 entries. + * + * Note that the parameters are *negated* log probabilities (and + * hence, are positive numbers), as is the return value. This is the + * key to the "fastness" of this function. + */ +LOGMATH_INLINE int +fast_logmath_add(logmath_t *lmath, int mlx, int mly) +{ + logadd_t *t = LOGMATH_TABLE(lmath); + int d, r; + + /* d must be positive, obviously. */ + if (mlx > mly) { + d = (mlx - mly); + r = mly; + } + else { + d = (mly - mlx); + r = mlx; + } + + return r - (((uint8 *)t->table)[d]); +} + +#endif /* __TIED_MGAU_COMMON_H__ */ diff --git a/include/tmat.h b/include/tmat.h new file mode 100644 index 0000000..5b64211 --- /dev/null +++ b/include/tmat.h @@ -0,0 +1,98 @@ +/* ==================================================================== + * Copyright (c) 1999-2004 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +#ifndef _S3_TMAT_H_ +#define _S3_TMAT_H_ + +#include +#include + +/** \file tmat.h + * \brief Transition matrix data structure. + */ +#ifdef __cplusplus +extern "C" { +#endif + +/** + * \struct tmat_t + * \brief Transition matrix data structure. All phone HMMs are assumed to have the same + * topology. + */ +typedef struct { + uint8 ***tp; /**< The transition matrices; kept in the same scale as acoustic scores; + tp[tmatid][from-state][to-state] */ + int16 n_tmat; /**< Number matrices */ + int16 n_state; /**< Number source states in matrix (only the emitting states); + Number destination states = n_state+1, it includes the exit state */ +} tmat_t; + + +/** Initialize transition matrix */ + +tmat_t *tmat_init (char const *tmatfile,/**< In: input file */ + logmath_t *lmath, /**< In: log math parameters */ + float64 tpfloor, /**< In: floor value for each non-zero transition probability */ + int32 breport /**< In: whether reporting the process of tmat_t */ + ); + + + +/** Dumping the transition matrix for debugging */ + +void tmat_dump (tmat_t *tmat, /**< In: transition matrix */ + FILE *fp /**< In: file pointer */ + ); + + +/** + * RAH, add code to remove memory allocated by tmat_init + */ + +void tmat_free (tmat_t *t /**< In: transition matrix */ + ); + +/** + * Report the detail of the transition matrix structure. + */ +void tmat_report(tmat_t *t /**< In: transition matrix*/ + ); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/vector.h b/include/vector.h new file mode 100644 index 0000000..ed81398 --- /dev/null +++ b/include/vector.h @@ -0,0 +1,89 @@ +/* ==================================================================== + * Copyright (c) 1999-2004 Carnegie Mellon University. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * This work was supported in part by funding from the Defense Advanced + * Research Projects Agency and the National Science Foundation of the + * United States of America, and the CMU Sphinx Speech Consortium. + * + * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND + * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY + * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ==================================================================== + * + */ + +/* + * vector.h -- vector routines. + * + * ********************************************** + * CMU ARPA Speech Project + * + * Copyright (c) 1997 Carnegie Mellon University. + * ALL RIGHTS RESERVED. + * ********************************************** + */ + + +#ifndef __VECTOR_H__ +#define __VECTOR_H__ + +/* System headers. */ +#include + +/* SphinxBase headers. */ +#include + +typedef float32 *vector_t; + +/* + * The reason for some of the "trivial" routines below is that they could be OPTIMIZED for SPEED + * at some point. + */ + + +/* Floor all elements of v[0..dim-1] to min value of f */ +void vector_floor(vector_t v, int32 dim, float64 f); + + +/* Floor all non-0 elements of v[0..dim-1] to min value of f */ +void vector_nz_floor(vector_t v, int32 dim, float64 f); + + +/* + * Normalize the elements of the given vector so that they sum to 1.0. If the sum is 0.0 + * to begin with, the vector is left untouched. Return value: The normalization factor. + */ +float64 vector_sum_norm(vector_t v, int32 dim); + + +/* Print vector in one line, in %11.4e format, terminated by newline */ +void vector_print(FILE *fp, vector_t v, int32 dim); + + +/* Return TRUE iff given vector is all 0.0 */ +int32 vector_is_zero (float32 *vec, /* In: Vector to be checked */ + int32 len); /* In: Length of above vector */ + +#endif /* VECTOR_H */ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 0000000..586bc64 --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add all the src cpp files with the headers +add_executable(featex.o featex.c ${HEADER_LIST}) + +# Ensure everything in `include` is included. +target_include_directories(featex.o PUBLIC ../include) +target_include_directories(featex.o PUBLIC "/usr/local/include/pocketsphinx") +target_include_directories(featex.o PUBLIC "/usr/local/include/sphinxbase") + +target_link_libraries(featex.o pocketsphinx sphinxbase m) + +# IDEs should put the headers in a nice place +source_group(TREE "${PROJECT_SOURCE_DIR}/include" PREFIX "Header Files" FILES ${HEADER_LIST}) diff --git a/featex.c b/src/featex.c similarity index 51% rename from featex.c rename to src/featex.c index 3fd7f06..fbba56e 100644 --- a/featex.c +++ b/src/featex.c @@ -10,6 +10,7 @@ #include #include "ps_alignment.h" + #include "state_align_search.h" #include "pocketsphinx_internal.h" #include "ps_search.h" @@ -17,8 +18,7 @@ #include #include -int -main(int argc, char *argv[]) +int main(int argc, char *argv[]) { ps_decoder_t *ps; dict_t *dict; @@ -43,58 +43,74 @@ main(int argc, char *argv[]) hash_table_t *hyptbl; double frated; - struct { + struct + { int start, dur, cipid, score; - } *algn; + } * algn; - if (argc < 2 || (*argv[1] == '-' && (*(argv[1]+1) != 'p' || argc < 3))) { + if (argc < 2 || (*argv[1] == '-' && (*(argv[1] + 1) != 'p' || argc < 3))) + { fprintf(stderr, "usage: %s [-p[u][w][p][t][d]] word....\n" - "-p: play [u]tterance, [w]ord(s), [p]honemes (default), " - "[t]riphones, and/or [d]iphones.\n", argv[0]); + "-p: play [u]tterance, [w]ord(s), [p]honemes (default), " + "[t]riphones, and/or [d]iphones.\n", + argv[0]); return 1; } play = 0; // by default play nothing - if (*argv[1] == '-' && *(argv[1]+1) == 'p') { + if (*argv[1] == '-' && *(argv[1] + 1) == 'p') + { play = 4; // just '-p' means to only play phonemes - if (*(argv[1]+2)) { + if (*(argv[1] + 2)) + { play = 0; - p = argv[1]+1; - while (*++p) { - if (*p == 'u') play |= 1; // utterance - else if (*p == 'w') play |= 2; // word(s) - else if (*p == 'p') play |= 4; // phonemes - else if (*p == 't') play |= 8; // triphones - else if (*p == 'd') play |= 16; // diphones - else { + p = argv[1] + 1; + while (*++p) + { + if (*p == 'u') + play |= 1; // utterance + else if (*p == 'w') + play |= 2; // word(s) + else if (*p == 'p') + play |= 4; // phonemes + else if (*p == 't') + play |= 8; // triphones + else if (*p == 'd') + play |= 16; // diphones + else + { fprintf(stderr, "%s: unrecogized -p option selection;\n" - "-p: play [u]tterance, [w]ord(s), [p]honemes (default)," - " [t]riphones, and/or [d]iphones.\n", argv[0]); + "-p: play [u]tterance, [w]ord(s), [p]honemes (default)," + " [t]riphones, and/or [d]iphones.\n", + argv[0]); return 1; } } } i = 2; - } else { + } + else + { i = 1; } #define FPS (16000 / FRATE * 2) sprintf(frates, "%d", FRATE); - frated = (double) FRATE; + frated = (double)FRATE; config = cmd_ln_init(NULL, ps_args(), FALSE, - "-hmm", MODELDIR, - "-dict", DICTNAME, - "-samprate", "16000", - "-topn", "64", // TODO parameterize for proper optimization - "-beam", "1e-57", - "-wbeam", "1e-56", - "-maxhmmpf", "-1", - "-frate", frates, - "-fsgusefiller", "no", - NULL); - if (!(ps = ps_init(config))) { + "-hmm", MODELDIR, + "-dict", DICTNAME, + "-samprate", "16000", + "-topn", "64", // TODO parameterize for proper optimization + "-beam", "1e-57", + "-wbeam", "1e-56", + "-maxhmmpf", "-1", + "-frate", frates, + "-fsgusefiller", "no", + NULL); + if (!(ps = ps_init(config))) + { fprintf(stderr, "%s: ps_init() failed.\n", argv[0]); return 2; } @@ -105,9 +121,11 @@ main(int argc, char *argv[]) al = ps_alignment_init(d2p); ps_alignment_add_word(al, dict_wordid(dict, ""), 0); - while (i < argc) { + while (i < argc) + { n = dict_wordid(dict, argv[i]); - if (n < 0) { + if (n < 0) + { fprintf(stderr, "%s: unrecogized word: %s\n", argv[0], argv[i]); return 3; } @@ -120,7 +138,8 @@ main(int argc, char *argv[]) search = state_align_search_init("state_align", config, acmod, al); rawfh = fopen(INFILENAME, "rb"); - if (!rawfh) { + if (!rawfh) + { fprintf(stderr, "%s: can't open audio input file: %s\n", argv[0], INFILENAME); return 4; @@ -129,33 +148,37 @@ main(int argc, char *argv[]) sz = ftell(rawfh); fbuf = fbip = malloc(sz); rewind(rawfh); - while (!feof(rawfh)) { - nread = fread(buf, sizeof(*buf), 2048, rawfh); - memcpy(fbip, buf, nread * sizeof(*buf)); - fbip += nread * sizeof(*buf); + while (!feof(rawfh)) + { + nread = fread(buf, sizeof(*buf), 2048, rawfh); + memcpy(fbip, buf, nread * sizeof(*buf)); + fbip += nread * sizeof(*buf); } acmod_start_utt(acmod); ps_search_start(search); - bptr = (const int16 *) fbuf; + bptr = (const int16 *)fbuf; nread = (fbip - fbuf) / sizeof(*buf); - while ((nfr = acmod_process_raw(acmod, &bptr, &nread, TRUE)) > 0) { - while (acmod->n_feat_frame > 0) { - ps_search_step(search, acmod->output_frame); - acmod_advance(acmod); - } - fprintf(stderr, "%s: processed %d frames\n", argv[0], nfr); + while ((nfr = acmod_process_raw(acmod, &bptr, &nread, TRUE)) > 0) + { + while (acmod->n_feat_frame > 0) + { + ps_search_step(search, acmod->output_frame); + acmod_advance(acmod); + } + fprintf(stderr, "%s: processed %d frames\n", argv[0], nfr); } acmod_end_utt(acmod); ps_search_finish(search); fprintf(stderr, "%s: aligned %d words, %d phones, and %d states\n", - argv[0], ps_alignment_n_words(al), ps_alignment_n_phones(al), - ps_alignment_n_states(al)); + argv[0], ps_alignment_n_words(al), ps_alignment_n_phones(al), + ps_alignment_n_states(al)); - if (play & 1) { // play utterance + if (play & 1) + { // play utterance rawfh = fopen("/tmp/outphone.raw", "wb"); fwrite(fbuf, sz, 1, rawfh); fclose(rawfh); @@ -171,12 +194,14 @@ main(int argc, char *argv[]) maxdur = 0; itor = ps_alignment_words(al); - while (itor) { + while (itor) + { ae = ps_alignment_iter_get(itor); fprintf(stderr, "%s: word '%s': %.2fs for %.2fs, score %d\n", argv[0], - dict->word[ae->id.wid].word, ae->start / frated, - ae->duration / frated, ae->score); - if (play & 2) { // play words + dict->word[ae->id.wid].word, ae->start / frated, + ae->duration / frated, ae->score); + if (play & 2) + { // play words rawfh = fopen("/tmp/outphone.raw", "wb"); fwrite(obuf, 8000, 1, rawfh); fwrite(fbuf + ae->start * FPS, ae->duration * FPS, 1, rawfh); @@ -187,12 +212,14 @@ main(int argc, char *argv[]) } itor2 = ps_alignment_iter_down(itor); wend = ae->duration + ae->start; - while (itor2) { + while (itor2) + { ae = ps_alignment_iter_get(itor2); - if (ae->start >= wend) break; + if (ae->start >= wend) + break; fprintf(stderr, "%s: sub-phone '%s': %.2fs for %.2fs, score %d\n", - argv[0], mdef->ciname[ae->id.pid.cipid], ae->start / frated, - ae->duration / frated, ae->score); + argv[0], mdef->ciname[ae->id.pid.cipid], ae->start / frated, + ae->duration / frated, ae->score); algn[n].start = ae->start; algn[n].dur = ae->duration; algn[n].score = ae->score; @@ -210,14 +237,16 @@ main(int argc, char *argv[]) obuf = malloc(16000 + FPS * maxdur * 3); memset(obuf, 0, 8000); - for (i = 0; i < n; i++) { + for (i = 0; i < n; i++) + { memcpy(obuf + 8000, fbuf + algn[i].start * FPS, algn[i].dur * FPS); memset(obuf + 8000 + algn[i].dur * FPS, 0, 8000); fprintf(stderr, "%s: phoneme %d: %s %.2fs for %.2fs, score %d\n", - argv[0], i + 1, mdef->ciname[algn[i].cipid], - algn[i].start / frated, algn[i].dur / frated, algn[i].score); - if (play & 4) { // play phonemes (default if '-p' specified) + argv[0], i + 1, mdef->ciname[algn[i].cipid], + algn[i].start / frated, algn[i].dur / frated, algn[i].score); + if (play & 4) + { // play phonemes (default if '-p' specified) rawfh = fopen("/tmp/outphone.raw", "wb"); fwrite(obuf, 16000 + algn[i].dur * FPS, 1, rawfh); fclose(rawfh); @@ -228,28 +257,30 @@ main(int argc, char *argv[]) hyptbl = hash_table_new(175, HASH_CASE_YES); // for hypothesis deduplication - for (i = 1; i < n; i++) { + for (i = 1; i < n; i++) + { - if (i == n-1) goto lastdiphone; + if (i == n - 1) + goto lastdiphone; - if (i > 1) printf(" "); + if (i > 1) + printf(" "); printf("%.2f %.3f", algn[i].dur / frated, 1 / log(2 - algn[i].score)); - memcpy(obuf + 8000, fbuf + algn[i-1].start * FPS, - (algn[i-1].dur + algn[i].dur + algn[i+1].dur) * FPS); - memset(obuf + 8000 - + (algn[i-1].dur + algn[i].dur + algn[i+1].dur) * FPS, + memcpy(obuf + 8000, fbuf + algn[i - 1].start * FPS, + (algn[i - 1].dur + algn[i].dur + algn[i + 1].dur) * FPS); + memset(obuf + 8000 + (algn[i - 1].dur + algn[i].dur + algn[i + 1].dur) * FPS, 0, 8000); fprintf(stderr, "%s: triphone %d: %s-%s-%s\n", argv[0], i, - mdef->ciname[algn[i-1].cipid], - mdef->ciname[algn[i].cipid], - mdef->ciname[algn[i+1].cipid]); - if (play & 8) { // play triphones + mdef->ciname[algn[i - 1].cipid], + mdef->ciname[algn[i].cipid], + mdef->ciname[algn[i + 1].cipid]); + if (play & 8) + { // play triphones rawfh = fopen("/tmp/outphone.raw", "wb"); - fwrite(obuf, 16000 + (algn[i-1].dur + algn[i].dur + - algn[i+1].dur) * FPS, 1, rawfh); + fwrite(obuf, 16000 + (algn[i - 1].dur + algn[i].dur + algn[i + 1].dur) * FPS, 1, rawfh); fclose(rawfh); system("play -q -r16k -ts16 -c1 /tmp/outphone.raw"); remove("/tmp/outphone.raw"); @@ -257,25 +288,31 @@ main(int argc, char *argv[]) grammar[0] = '\0'; strcat(grammar, "#JSGF V1.0;\ngrammar subalts;\npublic = sil1 "); - if (algn[i-1].cipid != mdef->sil) { - p = mdef->ciname[algn[i-1].cipid]; - q = grammar; - while (*++q); - while (*p) *q++ = tolower(*p++); - *q++ = '2'; - *q = '\0'; + if (algn[i - 1].cipid != mdef->sil) + { + p = mdef->ciname[algn[i - 1].cipid]; + q = grammar; + while (*++q) + ; + while (*p) + *q++ = tolower(*p++); + *q++ = '2'; + *q = '\0'; } strcat(grammar, " [ aa3 | ae3 | ah3 | ao3 | aw3 | ay3 | b3 | ch3 | d3" - " | dh3 | eh3 | er3 | ey3 | f3 | g3 | hh3 | ih3 | iy3 | jh3" - " | k3 | l3 | m3 | n3 | ng3 | ow3 | oy3 | p3 | r3 | s3 | sh3" - " | sil3 | t3 | th3 | uh3 | uw3 | v3 | w3 | y3 | z3 | zh3 ] "); - if (algn[i+1].cipid != mdef->sil) { - p = mdef->ciname[algn[i+1].cipid]; - q = grammar; - while (*++q); - while (*p) *q++ = tolower(*p++); - *q++ = '4'; - *q = '\0'; + " | dh3 | eh3 | er3 | ey3 | f3 | g3 | hh3 | ih3 | iy3 | jh3" + " | k3 | l3 | m3 | n3 | ng3 | ow3 | oy3 | p3 | r3 | s3 | sh3" + " | sil3 | t3 | th3 | uh3 | uw3 | v3 | w3 | y3 | z3 | zh3 ] "); + if (algn[i + 1].cipid != mdef->sil) + { + p = mdef->ciname[algn[i + 1].cipid]; + q = grammar; + while (*++q) + ; + while (*p) + *q++ = tolower(*p++); + *q++ = '4'; + *q = '\0'; } strcat(grammar, " sil5 ;\n"); @@ -284,33 +321,43 @@ main(int argc, char *argv[]) ps_set_jsgf_string(ps, "subalts", grammar); ps_set_search(ps, "subalts"); ps_start_utt(ps); - ps_process_raw(ps, (const int16 *) obuf, 8000 + // samples not bytes - (algn[i-1].dur + algn[i].dur + algn[i+1].dur) * 160, + ps_process_raw(ps, (const int16 *)obuf, 8000 + // samples not bytes + (algn[i - 1].dur + algn[i].dur + algn[i + 1].dur) * 160, FALSE, TRUE); ps_end_utt(ps); nb = ps_nbest(ps); j = found = 0; - target[0] = ' '; target[1] = '\0'; + target[0] = ' '; + target[1] = '\0'; strcat(target, mdef->ciname[algn[i].cipid]); strcat(target, "3"); p = target; - while (*++p) { *p = tolower(*p); } - while (nb) { - p = (char *) ps_nbest_hyp(nb, &score); - if (p) { // some hypotheses are literally NULL + while (*++p) + { + *p = tolower(*p); + } + while (nb) + { + p = (char *)ps_nbest_hyp(nb, &score); + if (p) + { // some hypotheses are literally NULL q = p; - while (*++q); - if (*(q-1) == '5') { // ignore hypotheses w/o whole match + while (*++q) + ; + if (*(q - 1) == '5') + { // ignore hypotheses w/o whole match // ignore repeated hypotheses - if (hash_table_lookup(hyptbl, p, NULL) == -1) { + if (hash_table_lookup(hyptbl, p, NULL) == -1) + { j++; fprintf(stderr, "%s: triphone hypothesis %d: %s, %d\n", - argv[0], j, p, score); + argv[0], j, p, score); hash_table_enter_int32(hyptbl, p, score); - if (strstr(p, target)) { + if (strstr(p, target)) + { found++; ps_nbest_free(nb); break; @@ -320,24 +367,26 @@ main(int argc, char *argv[]) } nb = ps_nbest_next(nb); } - if (!found) j = 42; // zero for bad recognition results or no match + if (!found) + j = 42; // zero for bad recognition results or no match fprintf(stderr, "%s: SUBSTITUTION: %.3f\n", argv[0], (42.0 - j) / 42.0); printf(" %.3f", (42.0 - j) / 42.0); hash_table_empty(hyptbl); - lastdiphone: // goto target for the final set of two phonemes + lastdiphone: // goto target for the final set of two phonemes - memcpy(obuf + 8000, fbuf + algn[i-1].start * FPS, - (algn[i-1].dur + algn[i].dur) * FPS); - memset(obuf + 8000 + (algn[i-1].dur + algn[i].dur) * FPS, 0, 8000); + memcpy(obuf + 8000, fbuf + algn[i - 1].start * FPS, + (algn[i - 1].dur + algn[i].dur) * FPS); + memset(obuf + 8000 + (algn[i - 1].dur + algn[i].dur) * FPS, 0, 8000); fprintf(stderr, "%s: diphone %d: %s-%s\n", argv[0], i, - mdef->ciname[algn[i-1].cipid], + mdef->ciname[algn[i - 1].cipid], mdef->ciname[algn[i].cipid]); - if (play & 16) { // play diphones + if (play & 16) + { // play diphones rawfh = fopen("/tmp/outphone.raw", "wb"); - fwrite(obuf, 16000 + (algn[i-1].dur + algn[i].dur) * FPS, 1, rawfh); + fwrite(obuf, 16000 + (algn[i - 1].dur + algn[i].dur) * FPS, 1, rawfh); fclose(rawfh); system("play -q -r16k -ts16 -c1 /tmp/outphone.raw"); remove("/tmp/outphone.raw"); @@ -345,53 +394,70 @@ main(int argc, char *argv[]) grammar[0] = '\0'; strcat(grammar, - "#JSGF V1.0;\ngrammar insdels;\npublic = sil1 [ "); - p = mdef->ciname[algn[i-1].cipid]; + "#JSGF V1.0;\ngrammar insdels;\npublic = sil1 [ "); + p = mdef->ciname[algn[i - 1].cipid]; q = grammar; - while (*++q); - while (*p) *q++ = tolower(*p++); + while (*++q) + ; + while (*p) + *q++ = tolower(*p++); *q++ = '2'; *q = '\0'; r = q; strcat(grammar, " ] [ aa3| ae3 | ah3 | ao3 | aw3 | ay3 | b3 | ch3" - " | d3 | dh3 | eh3 | er3 | ey3 | f3 | g3 | hh3 | ih3 | iy3" - " | jh3 | k3 | l3 | m3 | n3 | ng3 | ow3 | oy3 | p3 | r3 " - " | s3 | sh3 | sil3 | t3 | th3 | uh3 | uw3 | v3 | w3 | y3 " - " | z3 | zh3 ] "); - p = mdef->ciname[algn[i-1].cipid]; // first in diphone - while (*++q) { // blank out expected phoneme from possible insertions - if (isalpha(*q)) { - if ((*q == tolower(*p)) - && (((*(q+1) == '3') && *(p+1) == '\0') - || *(q+1) == tolower(*(p+1)))) { - *(q-2) = ' '; // blank out preceding '|' - *q = ' '; *(q+1) = ' '; *(q+2) = ' '; *(q+3) = ' '; - } else { + " | d3 | dh3 | eh3 | er3 | ey3 | f3 | g3 | hh3 | ih3 | iy3" + " | jh3 | k3 | l3 | m3 | n3 | ng3 | ow3 | oy3 | p3 | r3 " + " | s3 | sh3 | sil3 | t3 | th3 | uh3 | uw3 | v3 | w3 | y3 " + " | z3 | zh3 ] "); + p = mdef->ciname[algn[i - 1].cipid]; // first in diphone + while (*++q) + { // blank out expected phoneme from possible insertions + if (isalpha(*q)) + { + if ((*q == tolower(*p)) && (((*(q + 1) == '3') && *(p + 1) == '\0') || *(q + 1) == tolower(*(p + 1)))) + { + *(q - 2) = ' '; // blank out preceding '|' + *q = ' '; + *(q + 1) = ' '; + *(q + 2) = ' '; + *(q + 3) = ' '; + } + else + { q += 3; // advance past the rest of the phoneme } } } p = mdef->ciname[algn[i].cipid]; // second in diphone q = r; - while (*++q) { // blank out expected phoneme from possible insertions - if (isalpha(*q)) { - if ((*q == tolower(*p)) - && (((*(q+1) == '3') && *(p+1) == '\0') - || *(q+1) == tolower(*(p+1)))) { - *(q-2) = ' '; // blank out preceding '|' - *q = ' '; *(q+1) = ' '; *(q+2) = ' '; *(q+3) = ' '; - } else { + while (*++q) + { // blank out expected phoneme from possible insertions + if (isalpha(*q)) + { + if ((*q == tolower(*p)) && (((*(q + 1) == '3') && *(p + 1) == '\0') || *(q + 1) == tolower(*(p + 1)))) + { + *(q - 2) = ' '; // blank out preceding '|' + *q = ' '; + *(q + 1) = ' '; + *(q + 2) = ' '; + *(q + 3) = ' '; + } + else + { q += 3; // advance past the rest of the phoneme } } } - if (algn[i].cipid != mdef->sil) { - p = mdef->ciname[algn[i].cipid]; - q = grammar; - while (*++q); - while (*p) *q++ = tolower(*p++); - *q++ = '4'; - *q = '\0'; + if (algn[i].cipid != mdef->sil) + { + p = mdef->ciname[algn[i].cipid]; + q = grammar; + while (*++q) + ; + while (*p) + *q++ = tolower(*p++); + *q++ = '4'; + *q = '\0'; } strcat(grammar, " sil5 ;\n"); @@ -400,30 +466,39 @@ main(int argc, char *argv[]) ps_set_jsgf_string(ps, "insdels", grammar); ps_set_search(ps, "insdels"); ps_start_utt(ps); - ps_process_raw(ps, (const int16 *) obuf, 8000 + // samples not bytes - (algn[i-1].dur + algn[i].dur) * 160, FALSE, TRUE); + ps_process_raw(ps, (const int16 *)obuf, 8000 + // samples not bytes + (algn[i - 1].dur + algn[i].dur) * 160, + FALSE, TRUE); ps_end_utt(ps); nb = ps_nbest(ps); j = k = found = 0; - while (nb) { - p = (char *) ps_nbest_hyp(nb, &score); - if (p) { // some hypotheses are literally NULL + while (nb) + { + p = (char *)ps_nbest_hyp(nb, &score); + if (p) + { // some hypotheses are literally NULL q = p; - while (*++q); - if (*(q-1) == '5') { // ignore hypotheses w/o whole match + while (*++q) + ; + if (*(q - 1) == '5') + { // ignore hypotheses w/o whole match // ignore repeated hypotheses - if (hash_table_lookup(hyptbl, p, NULL) == -1) { + if (hash_table_lookup(hyptbl, p, NULL) == -1) + { j++; fprintf(stderr, "%s: diphone hypothesis %d: %s, %d\n", - argv[0], j, p, score); + argv[0], j, p, score); hash_table_enter_int32(hyptbl, p, score); - if (!strstr(p, "2 ")) k++; - if (strstr(p, "3 ")) k++; + if (!strstr(p, "2 ")) + k++; + if (strstr(p, "3 ")) + k++; - if (strstr(p, "2 ") && !strstr(p, "3 ")) { + if (strstr(p, "2 ") && !strstr(p, "3 ")) + { found++; ps_nbest_free(nb); break; @@ -435,9 +510,11 @@ main(int argc, char *argv[]) } if (j == 0) k = 160; // zero for bad recognition results - else if (!found) { + else if (!found) + { k += 80; // add half the range if the preferred hypothesis missed - if (k > 160) k = 160; // clamp + if (k > 160) + k = 160; // clamp } fprintf(stderr, "%s: INS/DEL: %.3f\n", argv[0], (160.0 - k) / 160); printf(" %.3f", (160.0 - k) / 160.0); From 09532ee358f492d0fa2bdd8cc9233f6277aab945 Mon Sep 17 00:00:00 2001 From: Joshua Arulsamy Date: Tue, 26 May 2020 20:15:38 -0600 Subject: [PATCH 3/9] Added proper argparsing. Previosuly the locations for combo.dict and the input raw file were both hardcoded, now they are command line arguments. Adds a TON of flexibility. --- src/featex.c | 204 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 147 insertions(+), 57 deletions(-) diff --git a/src/featex.c b/src/featex.c index fbba56e..984c63e 100644 --- a/src/featex.c +++ b/src/featex.c @@ -3,10 +3,10 @@ // by James Salsman, July-August 2017 // released under the MIT open source license -#define INFILENAME "featex.raw" +// #define INFILENAME "featex.raw" #define FRATE 65 #define MODELDIR "/usr/local/share/pocketsphinx/model/en-us/en-us" -#define DICTNAME "combo.dict" +// #define DICTNAME "combo.dict" #include #include "ps_alignment.h" @@ -17,9 +17,125 @@ #include #include +#include +#include +#include -int main(int argc, char *argv[]) +#include +#include +#include + +const char *argp_program_version = + "Featex 0.1"; + +const char *argp_program_bug_address = + ""; + +/* This structure is used by main to communicate with parse_opt. */ +struct arguments +{ + char *COMBO; + char *INFILE; + int PLAY; + char *PHRASE; +}; + +/* + OPTIONS. Field 1 in ARGP. + Order of fields: {NAME, KEY, ARG, FLAGS, DOC}. +*/ +static struct argp_option options[] = + { + {"combo", 'c', "COMBO_PATH", 0, "Path to combo.dict."}, + {"infile", 'i', "INFILE_PATH", 0, "Path to input raw file."}, + {"utterance", 'u', 0, 0, "Toggle play utterances"}, + {"word", 'w', 0, 0, "Toggle play words"}, + {"phonemes", 'p', 0, 0, "Toggle play phonemes"}, + {"triphones", 't', 0, 0, "Toggle play triphones"}, + {"diphones", 'd', 0, 0, "Toggle play diphones"}, + {"phrase", 'P', "'QUOTE SURROUNDED PHRASE'", 0, "Input phrase"}, + {0}}; + +/* + PARSER. Field 2 in ARGP. + Order of parameters: KEY, ARG, STATE. +*/ +static error_t +parse_opt(int key, char *arg, struct argp_state *state) +{ + struct arguments *arguments = state->input; + + switch (key) + { + case 'c': + arguments->COMBO = arg; + break; + case 'i': + arguments->INFILE = arg; + break; + case 'P': + arguments->PHRASE = arg; + break; + case 'u': + arguments->PLAY |= 1; + break; + case 'w': + arguments->PLAY |= 2; + break; + case 'p': + arguments->PLAY |= 4; + break; + case 't': + arguments->PLAY |= 8; + break; + case 'd': + arguments->PLAY |= 16; + break; + + default: + return ARGP_ERR_UNKNOWN; + } + return 0; +} + +/* + DOC. Field 4 in ARGP. + Program documentation. +*/ +static char doc[] = + "featex -- PocketSphinx phonetic feature extraction for intelligibility prediction and remediation"; + +/* + The ARGP structure itself. +*/ +static struct argp argp = {options, parse_opt, 0, doc}; + +int main(int argc, char **argv) { + + struct arguments arguments; + + /* Set argument defaults */ + arguments.COMBO = "combo.dict"; + arguments.INFILE = "featex.raw"; + arguments.PHRASE = ""; + arguments.PLAY = 0; + + /* Where the magic happens */ + argp_parse(&argp, argc, argv, 0, 0, &arguments); + + if (arguments.PHRASE == "") + { + fprintf(stderr, "Missing phrase, ensure you used -P\n"); + exit(-1); + } + + /* Debug - Print argument values */ + // printf("alpha = %s\nbravo = %s\nPLAY = %i\nPHRASE = %s", + // arguments.COMBO, arguments.INFILE, arguments.PLAY, arguments.PHRASE); + + // exit(0); + ps_decoder_t *ps; dict_t *dict; dict2pid_t *d2p; @@ -34,7 +150,8 @@ int main(int argc, char *argv[]) char *fbuf, *fbip, *obuf; size_t nread; int16 const *bptr; - int sz, nfr, wend, n, maxdur, i, j, k, play, found; + int sz, nfr, wend, n, maxdur, i, j, k, found; + int play = arguments.PLAY; ps_alignment_entry_t *ae; char grammar[1000], target[10], frates[10]; char *p, *q, *r; // string manipulation pointers for constructing grammar @@ -48,59 +165,13 @@ int main(int argc, char *argv[]) int start, dur, cipid, score; } * algn; - if (argc < 2 || (*argv[1] == '-' && (*(argv[1] + 1) != 'p' || argc < 3))) - { - fprintf(stderr, "usage: %s [-p[u][w][p][t][d]] word....\n" - "-p: play [u]tterance, [w]ord(s), [p]honemes (default), " - "[t]riphones, and/or [d]iphones.\n", - argv[0]); - return 1; - } - - play = 0; // by default play nothing - if (*argv[1] == '-' && *(argv[1] + 1) == 'p') - { - play = 4; // just '-p' means to only play phonemes - if (*(argv[1] + 2)) - { - play = 0; - p = argv[1] + 1; - while (*++p) - { - if (*p == 'u') - play |= 1; // utterance - else if (*p == 'w') - play |= 2; // word(s) - else if (*p == 'p') - play |= 4; // phonemes - else if (*p == 't') - play |= 8; // triphones - else if (*p == 'd') - play |= 16; // diphones - else - { - fprintf(stderr, "%s: unrecogized -p option selection;\n" - "-p: play [u]tterance, [w]ord(s), [p]honemes (default)," - " [t]riphones, and/or [d]iphones.\n", - argv[0]); - return 1; - } - } - } - i = 2; - } - else - { - i = 1; - } - #define FPS (16000 / FRATE * 2) sprintf(frates, "%d", FRATE); frated = (double)FRATE; config = cmd_ln_init(NULL, ps_args(), FALSE, "-hmm", MODELDIR, - "-dict", DICTNAME, + "-dict", arguments.COMBO, "-samprate", "16000", "-topn", "64", // TODO parameterize for proper optimization "-beam", "1e-57", @@ -121,27 +192,46 @@ int main(int argc, char *argv[]) al = ps_alignment_init(d2p); ps_alignment_add_word(al, dict_wordid(dict, ""), 0); - while (i < argc) + + char words[50][50]; + int letter, cnt; + letter = cnt = 0; + for (i = 0; i < strlen(arguments.PHRASE); i++) { - n = dict_wordid(dict, argv[i]); + if (arguments.PHRASE[i] == ' ' || arguments.PHRASE[i] == '\0') + { + words[cnt][letter] = '\0'; + cnt++; //for next word + letter = 0; //for next word, init index to 0 + } + else + { + words[cnt][letter] = arguments.PHRASE[i]; + letter++; + } + } + + for (int i = 0; i < cnt; i++) + { + n = dict_wordid(dict, words[i]); if (n < 0) { - fprintf(stderr, "%s: unrecogized word: %s\n", argv[0], argv[i]); + fprintf(stderr, "%s: unrecogized word: %s\n", argv[0], words[i]); return 3; } ps_alignment_add_word(al, n, 0); - i++; } + ps_alignment_add_word(al, dict_wordid(dict, ""), 0); ps_alignment_populate(al); search = state_align_search_init("state_align", config, acmod, al); - rawfh = fopen(INFILENAME, "rb"); + rawfh = fopen(arguments.INFILE, "rb"); if (!rawfh) { fprintf(stderr, "%s: can't open audio input file: %s\n", - argv[0], INFILENAME); + argv[0], arguments.INFILE); return 4; } fseek(rawfh, 0L, SEEK_END); From 340b7a4aa65f630e1026e950517614b8a594c91e Mon Sep 17 00:00:00 2001 From: Joshua Arulsamy Date: Tue, 26 May 2020 20:24:21 -0600 Subject: [PATCH 4/9] Cleaned up * Removed extra headers * Reordered headers to be more readable. * Reorganized repository --- ...en-English-Intelligibility-Remediation.pdf | Bin .../because-00766t-2.wav | Bin .../because-01004t-5.wav | Bin combo.dict => assets/combo.dict | 0 featex-tran.txt => assets/featex-tran.txt | 0 featex.raw => assets/featex.raw | Bin example-for-fig3.py | 109 ----- include/allphone_search.h | 179 -------- include/blkarray_list.h | 139 ------ include/fsg_history.h | 215 --------- include/fsg_lextree.h | 255 ---------- include/fsg_search_internal.h | 153 ------ include/kws_detections.h | 76 --- include/kws_search.h | 142 ------ include/ms_gauden.h | 150 ------ include/ms_mgau.h | 143 ------ include/ms_senone.h | 131 ------ include/ngram_search.h | 434 ------------------ include/ngram_search_fwdflat.h | 81 ---- include/ngram_search_fwdtree.h | 83 ---- include/phone_loop_search.h | 102 ---- include/ps_lattice_internal.h | 282 ------------ include/ptm_mgau.h | 103 ----- include/s2_semi_mgau.h | 98 ---- include/tied_mgau_common.h | 121 ----- include/vector.h | 89 ---- src/featex.c | 21 +- 27 files changed, 6 insertions(+), 3100 deletions(-) rename Spoken-English-Intelligibility-Remediation.pdf => assets/Spoken-English-Intelligibility-Remediation.pdf (100%) rename because-00766t-2.wav => assets/because-00766t-2.wav (100%) rename because-01004t-5.wav => assets/because-01004t-5.wav (100%) rename combo.dict => assets/combo.dict (100%) rename featex-tran.txt => assets/featex-tran.txt (100%) rename featex.raw => assets/featex.raw (100%) delete mode 100644 example-for-fig3.py delete mode 100644 include/allphone_search.h delete mode 100644 include/blkarray_list.h delete mode 100644 include/fsg_history.h delete mode 100644 include/fsg_lextree.h delete mode 100644 include/fsg_search_internal.h delete mode 100644 include/kws_detections.h delete mode 100644 include/kws_search.h delete mode 100644 include/ms_gauden.h delete mode 100644 include/ms_mgau.h delete mode 100644 include/ms_senone.h delete mode 100644 include/ngram_search.h delete mode 100644 include/ngram_search_fwdflat.h delete mode 100644 include/ngram_search_fwdtree.h delete mode 100644 include/phone_loop_search.h delete mode 100644 include/ps_lattice_internal.h delete mode 100644 include/ptm_mgau.h delete mode 100644 include/s2_semi_mgau.h delete mode 100644 include/tied_mgau_common.h delete mode 100644 include/vector.h diff --git a/Spoken-English-Intelligibility-Remediation.pdf b/assets/Spoken-English-Intelligibility-Remediation.pdf similarity index 100% rename from Spoken-English-Intelligibility-Remediation.pdf rename to assets/Spoken-English-Intelligibility-Remediation.pdf diff --git a/because-00766t-2.wav b/assets/because-00766t-2.wav similarity index 100% rename from because-00766t-2.wav rename to assets/because-00766t-2.wav diff --git a/because-01004t-5.wav b/assets/because-01004t-5.wav similarity index 100% rename from because-01004t-5.wav rename to assets/because-01004t-5.wav diff --git a/combo.dict b/assets/combo.dict similarity index 100% rename from combo.dict rename to assets/combo.dict diff --git a/featex-tran.txt b/assets/featex-tran.txt similarity index 100% rename from featex-tran.txt rename to assets/featex-tran.txt diff --git a/featex.raw b/assets/featex.raw similarity index 100% rename from featex.raw rename to assets/featex.raw diff --git a/example-for-fig3.py b/example-for-fig3.py deleted file mode 100644 index c77a1cd..0000000 --- a/example-for-fig3.py +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env python2 -# -*- coding: utf-8 -*- -""" -Created on Sun Aug 20 - -@author: jsalsman -""" - -from keras.models import Sequential -from keras.layers import Dense, Dropout -from keras.utils.np_utils import to_categorical -from numpy import asarray -from scipy.stats import rankdata - -lines = [] -with open('featex-tran.txt', 'r') as f: # or, phrase-sliced.txt which has some - # overlapping words so maybe rename - # one of them if you want to use the - # same server for the phrase as for - # the 82 words - lines.extend(f.readlines()) - -X = [] # testing data independents -y = [] # testing data dependents -word = '' # word name -model = {} -n = 0 -layers = 4 -units = 32 -epochs = 1000 -drop = 0.25 -features = None - -for line in lines + ['.']: - tokens = line.strip().split() - if line[0] != ' ': # new word - if word != '': # not the first word - print ("word:", word, n, "transcripts,", features, "features") - - y_cat = to_categorical(y) - model[word].fit(X, y_cat, epochs=epochs, verbose=0) - - # now you can get the probability of intelligibility for some - # featex vector Z this way: - # pi = model[word].predict(asarray(Z).reshape(1, -1))[0][1] - - if line != '.': # not the last line - word = tokens[2] - features = int(tokens[4]) * int(tokens[6]) + int(tokens[8]) - X = []; y = []; n = 0 - - model[word] = Sequential() # DNN - model[word].add(Dense(units, input_dim=features, - activation='softmax', - kernel_initializer='glorot_uniform')) - model[word].add(Dropout(drop)) - for i in range(layers): - model[word].add(Dense(units, - kernel_initializer='glorot_uniform')) - model[word].add(Dropout(drop)) - model[word].add(Dense(2, activation='softmax', - kernel_initializer='glorot_uniform')) - model[word].compile(optimizer='adam', - loss='categorical_crossentropy') - - else: # read a transcription's word data observation - if len(tokens) > features + 1: # ignore incomplete recognition results - fvec = [] - for i in range(features): - fvec.append(float(tokens[i + 2])) - if tokens[1] == "<-": - X.append(fvec) - y.append(float(tokens[0])) - n += 1 - - -because_00766t_2 = [0.22, 0.178, 0.929, 0.744, 0.05, 0.200, 0.381, 0.981, - 0.05, 0.182, 0.548, 0.000, 0.25, 0.161, 0.786, 0.512, - 0.43, 0.150, 0.929, 0.869, 0.725] -# unintelligible, pronounced "cuz" without the "bee-" -# 0.049262498 - -because_01004t_5 = [0.08, 0.277, 1.000, 0.000, 0.09, 0.275, 0.976, 0.000, - 0.11, 0.261, 0.952, 0.988, 0.06, 0.198, 0.929, 1.000, - 0.05, 0.181, 0.333, 0.919, 0.569] -# intelligible, pronounced "because-ah" as in a typical Chinese primary ESL student accent -# 0.57494467 - -def perturb(V, word): - print(model[word].predict(asarray(V).reshape(1, -1))[0][1]) - phonemes = (len(V) - 1) // 4 - pbs = [] - for n in range(phonemes): - Z = list(V) - Z[n*4 + 1] *= 1.5 - Z[n*4 + 2] *= 1.5 - Z[n*4 + 3] *= 1.5 - p_i = model[word].predict(asarray(Z).reshape(1, -1))[0][1] - print(p_i) - pbs.append(p_i) - return [int(i) for i in rankdata(pbs)] - -model['because'].predict(asarray([because_00766t_2]).reshape(1, -1))[0][1] -# 0.049262498 - -model['because'].predict(asarray([because_01004t_5]).reshape(1, -1))[0][1] -# 0.57494467 - -perturb(because_00766t_2, 'because') diff --git a/include/allphone_search.h b/include/allphone_search.h deleted file mode 100644 index d09a4e3..0000000 --- a/include/allphone_search.h +++ /dev/null @@ -1,179 +0,0 @@ -/* -*- c-basic-offset:4; indent-tabs-mode: nil -*- */ -/* ==================================================================== - * Copyright (c) 2014 Carnegie Mellon University. All rights - * reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * - * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND - * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY - * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * ==================================================================== - * - */ - -/* - * allphone_search.h -- Search structures for phoneme decoding. - */ - - -#ifndef __ALLPHONE_SEARCH_H__ -#define __ALLPHONE_SEARCH_H__ - - -/* SphinxBase headers. */ -#include -#include -#include -#include - -/* Local headers. */ -#include "pocketsphinx_internal.h" -#include "blkarray_list.h" -#include "hmm.h" - -/** - * Models a single unique pair. - * Can represent several different triphones, but all with the same parent basephone. - * (NOTE: Word-position attribute of triphone is ignored.) - */ -typedef struct phmm_s { - hmm_t hmm; /**< Base HMM structure */ - s3pid_t pid; /**< Phone id (temp. during init.) */ - s3cipid_t ci; /**< Parent basephone for this PHMM */ - bitvec_t *lc; /**< Set (bit-vector) of left context phones seen for this PHMM */ - bitvec_t *rc; /**< Set (bit-vector) of right context phones seen for this PHMM */ - struct phmm_s *next; /**< Next unique PHMM for same parent basephone */ - struct plink_s *succlist; /**< List of predecessor PHMM nodes */ -} phmm_t; - -/** - * List of links from a PHMM node to its successors; one link per successor. - */ -typedef struct plink_s { - phmm_t *phmm; /**< Successor PHMM node */ - struct plink_s *next; /**< Next link for parent PHMM node */ -} plink_t; - -/** - * History (paths) information at any point in allphone Viterbi search. - */ -typedef struct history_s { - phmm_t *phmm; /**< PHMM ending this path */ - int32 score; /**< Path score for this path */ - int32 tscore; /**< Transition score for this path */ - frame_idx_t ef; /**< End frame */ - int32 hist; /**< Previous history entry */ -} history_t; - -/** - * Phone level segmentation information - */ -typedef struct phseg_s { - s3cipid_t ci; /* CI-phone id */ - frame_idx_t sf, ef; /* Start and end frame for this phone occurrence */ - int32 score; /* Acoustic score for this segment of alignment */ - int32 tscore; /* Transition ("LM") score for this segment */ -} phseg_t; - -/** - * Segment iterator over list of phseg - */ -typedef struct phseg_iter_s { - ps_seg_t base; - glist_t seg; -} phseg_iter_t; - -/** - * Implementation of allphone search structure. - */ -typedef struct allphone_search_s { - ps_search_t base; - - hmm_context_t *hmmctx; /**< HMM context. */ - ngram_model_t *lm; /**< Ngram model set */ - int32 ci_only; /**< Use context-independent phones for decoding */ - phmm_t **ci_phmm; /**< PHMM lists (for each CI phone) */ - int32 *ci2lmwid; /**< Mapping of CI phones to LM word IDs */ - - int32 beam, pbeam; /**< Effective beams after applying beam_factor */ - int32 lw, inspen; /**< Language weights */ - - frame_idx_t frame; /**< Current frame. */ - float32 ascale; /**< Acoustic score scale for posterior probabilities. */ - - int32 n_tot_frame; /**< Total number of frames processed */ - int32 n_hmm_eval; /**< Total HMMs evaluated this utt */ - int32 n_sen_eval; /**< Total senones evaluated this utt */ - - /* Backtrace information */ - blkarray_list_t *history; /**< List of history nodes allocated in each frame */ - /* Hypothesis DAG */ - glist_t segments; - - ptmr_t perf; /**< Performance counter */ - -} allphone_search_t; - -/** - * Create, initialize and return a search module. - */ -ps_search_t *allphone_search_init(const char *name, - ngram_model_t * lm, - cmd_ln_t * config, - acmod_t * acmod, - dict_t * dict, dict2pid_t * d2p); - -/** - * Deallocate search structure. - */ -void allphone_search_free(ps_search_t * search); - -/** - * Update allphone search module. - */ -int allphone_search_reinit(ps_search_t * search, dict_t * dict, - dict2pid_t * d2p); - -/** - * Prepare the allphone search structure for beginning decoding of the next - * utterance. - */ -int allphone_search_start(ps_search_t * search); - -/** - * Step one frame forward through the Viterbi search. - */ -int allphone_search_step(ps_search_t * search, int frame_idx); - -/** - * Windup and clean the allphone search structure after utterance. - */ -int allphone_search_finish(ps_search_t * search); - -/** - * Get hypothesis string from the allphone search. - */ -char const *allphone_search_hyp(ps_search_t * search, int32 * out_score); - -#endif /* __ALLPHONE_SEARCH_H__ */ diff --git a/include/blkarray_list.h b/include/blkarray_list.h deleted file mode 100644 index a2e8513..0000000 --- a/include/blkarray_list.h +++ /dev/null @@ -1,139 +0,0 @@ -/* ==================================================================== - * Copyright (c) 1999-2004 Carnegie Mellon University. All rights - * reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * - * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND - * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY - * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * ==================================================================== - * - */ - -/* - * blkarray_list.h -- array-based list structure, for memory and access - * efficiency. - * - * HISTORY - * - * $Log: blkarray_list.h,v $ - * Revision 1.1.1.1 2006/05/23 18:45:02 dhuggins - * re-importation - * - * Revision 1.2 2004/12/10 16:48:58 rkm - * Added continuous density acoustic model handling - * - * Revision 1.1 2004/07/16 00:57:12 egouvea - * Added Ravi's implementation of FSG support. - * - * Revision 1.2 2004/05/27 14:22:57 rkm - * FSG cross-word triphones completed (but for single-phone words) - * - * Revision 1.1.1.1 2004/03/01 14:30:31 rkm - * - * - * Revision 1.1 2004/02/26 01:14:48 rkm - * *** empty log message *** - * - * - * 18-Feb-2004 M K Ravishankar (rkm@cs.cmu.edu) at Carnegie Mellon - * Started. - */ - - -#ifndef __S2_BLKARRAY_LIST_H__ -#define __S2_BLKARRAY_LIST_H__ - - -#include - - -/* - * For maintaining a (conceptual) "list" of pointers to arbitrary data. - * The application is responsible for knowing the true data type. - * Use an array instead of a true list for efficiency (both memory and - * speed). But use a blocked (2-D) array to allow dynamic resizing at a - * coarse grain. An entire block is allocated or freed, as appropriate. - */ -typedef struct blkarray_list_s { - void ***ptr; /* ptr[][] is the user-supplied ptr */ - int32 maxblks; /* size of ptr (#rows) */ - int32 blksize; /* size of ptr[] (#cols, ie, size of each row) */ - int32 n_valid; /* # entries actually stored in the list */ - int32 cur_row; /* The current row being that has empty entry */ - int32 cur_row_free; /* First entry valid within the current row */ -} blkarray_list_t; - -/* Access macros */ -#define blkarray_list_ptr(l,r,c) ((l)->ptr[r][c]) -#define blkarray_list_maxblks(l) ((l)->maxblks) -#define blkarray_list_blksize(l) ((l)->blksize) -#define blkarray_list_n_valid(l) ((l)->n_valid) -#define blkarray_list_cur_row(l) ((l)->cur_row) -#define blkarray_list_cur_row_free(l) ((l)->cur_row_free) - - -/* - * Initialize and return a new blkarray_list containing an empty list - * (i.e., 0 length). Sized for the given values of maxblks and blksize. - * NOTE: (maxblks * blksize) should not overflow int32, but this is not - * checked. - * Return the allocated entry if successful, NULL if any error. - */ -blkarray_list_t *_blkarray_list_init (int32 maxblks, int32 blksize); - - -/* - * Like _blkarray_list_init() above, but for some default values of - * maxblks and blksize. - */ -blkarray_list_t *blkarray_list_init ( void ); - -/** - * Completely finalize a blkarray_list. - */ -void blkarray_list_free(blkarray_list_t *bl); - - -/* - * Append the given new entry (data) to the end of the list. - * Return the index of the entry if successful, -1 if any error. - * The returned indices are guaranteed to be successive integers (i.e., - * 0, 1, 2...) for successive append operations, until the list is reset, - * when they resume from 0. - */ -int32 blkarray_list_append (blkarray_list_t *, void *data); - - -/* - * Free all the entries in the list (using ckd_free) and reset the - * list length to 0. - */ -void blkarray_list_reset (blkarray_list_t *); - - -/* Gets n-th element of the array list */ -void * blkarray_list_get(blkarray_list_t *, int32 n); - -#endif diff --git a/include/fsg_history.h b/include/fsg_history.h deleted file mode 100644 index 5eaad65..0000000 --- a/include/fsg_history.h +++ /dev/null @@ -1,215 +0,0 @@ -/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ -/* ==================================================================== - * Copyright (c) 1999-2004 Carnegie Mellon University. All rights - * reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * - * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND - * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY - * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * ==================================================================== - * - */ -/* - * fsg_history.h -- FSG Viterbi decode history - * - * ********************************************** - * CMU ARPA Speech Project - * - * Copyright (c) 1999 Carnegie Mellon University. - * ALL RIGHTS RESERVED. - * ********************************************** - * - * HISTORY - * - * $Log: fsg_history.h,v $ - * Revision 1.1.1.1 2006/05/23 18:45:02 dhuggins - * re-importation - * - * Revision 1.1 2004/07/16 00:57:12 egouvea - * Added Ravi's implementation of FSG support. - * - * Revision 1.7 2004/07/07 22:30:35 rkm - * *** empty log message *** - * - * Revision 1.6 2004/07/07 13:56:33 rkm - * Added reporting of (acoustic score - best senone score)/frame - * - * Revision 1.5 2004/06/25 14:49:08 rkm - * Optimized size of history table and speed of word transitions by maintaining only best scoring word exits at each state - * - * Revision 1.4 2004/06/23 20:32:16 rkm - * *** empty log message *** - * - * Revision 1.3 2004/05/27 15:16:08 rkm - * *** empty log message *** - * - * - * 25-Feb-2004 M K Ravishankar (rkm@cs.cmu.edu) at Carnegie Mellon University - * Started, based on S3.3 version. - */ - - -#ifndef __S2_FSG_HISTORY_H__ -#define __S2_FSG_HISTORY_H__ - - -/* SphinxBase headers. */ -#include -#include - -/* Local headers. */ -#include "blkarray_list.h" -#include "fsg_lextree.h" -#include "dict.h" - -/* - * The Viterbi history structure. This is a tree, with the root at the - * FSG start state, at frame 0, with a null predecessor. - */ - -/* - * A single Viterbi history entry - */ -typedef struct fsg_hist_entry_s { - fsg_link_t *fsglink; /* Link taken result in this entry */ - int32 score; /* Total path score at the end of this - transition */ - int32 pred; /* Predecessor entry; -1 if none */ - frame_idx_t frame; /* Ending frame for this entry */ - int16 lc; /* Left context provided by this entry to - succeeding words */ - fsg_pnode_ctxt_t rc; /* Possible right contexts to which this entry - applies */ -} fsg_hist_entry_t; - -/* Access macros */ -#define fsg_hist_entry_fsglink(v) ((v)->fsglink) -#define fsg_hist_entry_frame(v) ((v)->frame) -#define fsg_hist_entry_score(v) ((v)->score) -#define fsg_hist_entry_pred(v) ((v)->pred) -#define fsg_hist_entry_lc(v) ((v)->lc) -#define fsg_hist_entry_rc(v) ((v)->rc) - - -/* - * The entire tree of history entries (fsg_history_t.entries). - * Optimization: In a given frame, there may be several history entries, with - * the same left and right phonetic context, terminating in a particular state. - * Only the best scoring one of these needs to be saved, since everything else - * will be pruned according to the Viterbi algorithm. frame_entries is used - * temporarily in each frame to determine these best scoring entries in that - * frame. Only the ones not pruned are transferred to entries at the end of - * the frame. However, null transitions are a problem since they create - * entries that depend on entries created in the CURRENT frame. Hence, this - * pruning is done in two stages: first for the non-null transitions, and then - * for the null transitions alone. (This solution is sub-optimal, and can be - * improved with a little more work. SMOP.) - * Why is frame_entries a list? Each entry has a unique terminating state, - * and has a unique lc CIphone. But it has a SET of rc CIphones. - * frame_entries[s][lc] is an ordered list of entries created in the current - * frame, terminating in state s, and with left context lc. The list is in - * descending order of path score. When a new entry with (s,lc) arrives, - * its position in the list is determined. Then its rc set is modified by - * subtracting the union of the rc's of all its predecessors (i.e., better - * scoring entries). If the resulting rc set is empty, the entry is discarded. - * Otherwise, it is inserted, and the rc sets of all downstream entries in the - * list are updated by subtracting the new entry's rc. If any of them becomes - * empty, it is also discarded. - * As mentioned earlier, this procedure is applied in two stages, for the - * non-null transitions, and the null transitions, separately. - */ -typedef struct fsg_history_s { - fsg_model_t *fsg; /* The FSG for which this object applies */ - blkarray_list_t *entries; /* A list of history table entries; the root - entry is the first element of the list */ - glist_t **frame_entries; - int n_ciphone; -} fsg_history_t; - - -/* - * One-time intialization: Allocate and return an initially empty history - * module. - */ -fsg_history_t *fsg_history_init(fsg_model_t *fsg, dict_t *dict); - -void fsg_history_utt_start(fsg_history_t *h); - -void fsg_history_utt_end(fsg_history_t *h); - - -/* - * Create a history entry recording the completion of the given FSG - * transition, at the end of the given frame, with the given score, and - * the given predecessor history entry. - * The entry is initially temporary, and may be superseded by another - * with a higher score. The surviving entries must be transferred to - * the main history table, via fsg_history_end_frame(). - */ -void fsg_history_entry_add (fsg_history_t *h, - fsg_link_t *l, /* FSG transition */ - int32 frame, - int32 score, - int32 pred, - int32 lc, - fsg_pnode_ctxt_t rc); - -/* - * Transfer the surviving history entries for this frame into the permanent - * history table. This function can be called several times during a frame. - * Each time, the entries surviving so far are transferred, and the temporary - * lists cleared. This feature is used to handle the entries due to non-null - * transitions and null transitions separately. - */ -void fsg_history_end_frame (fsg_history_t *h); - - -/* Clear the hitory table */ -void fsg_history_reset (fsg_history_t *h); - - -/* Return the number of valid entries in the given history table */ -int32 fsg_history_n_entries (fsg_history_t *h); - -/* - * Return a ptr to the history entry for the given ID; NULL if there is no - * such entry. - */ -fsg_hist_entry_t *fsg_history_entry_get(fsg_history_t *h, int32 id); - - -/* - * Switch the FSG associated with the given history module. Should be done - * when the history list is empty. If not empty, the list is cleared. - */ -void fsg_history_set_fsg (fsg_history_t *h, fsg_model_t *fsg, dict_t *dict); - -/* Free the given Viterbi search history object */ -void fsg_history_free (fsg_history_t *h); - -/* Print the entire history */ -void fsg_history_print(fsg_history_t *h, dict_t *dict); - -#endif diff --git a/include/fsg_lextree.h b/include/fsg_lextree.h deleted file mode 100644 index 563065c..0000000 --- a/include/fsg_lextree.h +++ /dev/null @@ -1,255 +0,0 @@ -/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ -/* ==================================================================== - * Copyright (c) 1999-2013 Carnegie Mellon University. All rights - * reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * - * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND - * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY - * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * ==================================================================== - * - */ -/* - * fsg_lextree.h -- The collection of all the lextrees for the entire FSM. - * - */ - -#ifndef __S2_FSG_LEXTREE_H__ -#define __S2_FSG_LEXTREE_H__ - -/* SphinxBase headers. */ -#include -#include - -/* Local headers. */ -#include "hmm.h" -#include "dict.h" -#include "dict2pid.h" - -/* - * Compile-time constant determining the size of the - * bitvector fsg_pnode_t.fsg_pnode_ctxt_t.bv. (See below.) - * But it makes memory allocation simpler and more efficient. - * Make it smaller (2) to save memory if your phoneset has less than - * 64 phones. - */ -#define FSG_PNODE_CTXT_BVSZ 4 - -typedef struct { - uint32 bv[FSG_PNODE_CTXT_BVSZ]; -} fsg_pnode_ctxt_t; - - -/* - * All transitions (words) out of any given FSG state represented are by a - * phonetic prefix lextree (except for epsilon or null transitions; they - * are not part of the lextree). Lextree leaf nodes represent individual - * FSG transitions, so no sharing is allowed at the leaf nodes. The FSG - * transition probs are distributed along the lextree: the prob at a node - * is the max of the probs of all leaf nodes (and, hence, FSG transitions) - * reachable from that node. - * - * To conserve memory, the underlying HMMs with state-level information are - * allocated only as needed. Root and leaf nodes must also account for all - * the possible phonetic contexts, with an independent HMM for each distinct - * context. - */ -typedef struct fsg_pnode_s { - /* - * If this is not a leaf node, the first successor (child) node. Otherwise - * the parent FSG transition for which this is the leaf node (for figuring - * the FSG destination state, and word emitted by the transition). A node - * may have several children. The succ ptr gives just the first; the rest - * are linked via the sibling ptr below. - */ - union { - struct fsg_pnode_s *succ; - fsg_link_t *fsglink; - } next; - - /* - * For simplicity of memory management (i.e., freeing the pnodes), all - * pnodes allocated for all transitions out of a state are maintained in a - * linear linked list through the alloc_next pointer. - */ - struct fsg_pnode_s *alloc_next; - - /* - * The next node that is also a child of the parent of this node; NULL if - * none. - */ - struct fsg_pnode_s *sibling; - - /* - * The transition (log) probability to be incurred upon transitioning to - * this node. (Transition probabilities are really associated with the - * transitions. But a lextree node has exactly one incoming transition. - * Hence, the prob can be associated with the node.) - * This is a logs2(prob) value, and includes the language weight. - */ - int32 logs2prob; - - /* - * The root and leaf positions associated with any transition have to deal - * with multiple phonetic contexts. However, different contexts may result - * in the same SSID (senone-seq ID), and can share a single pnode with that - * SSID. But the pnode should track the set of context CI phones that share - * it. Hence the fsg_pnode_ctxt_t bit-vector set-representation. (For - * simplicity of implementation, its size is a compile-time constant for - * now.) Single phone words would need a 2-D array of context, but that's - * too expensive. For now, they simply use SIL as right context, so only - * the left context is properly modelled. - * (For word-internal phones, this field is unused, of course.) - */ - fsg_pnode_ctxt_t ctxt; - - uint16 ci_ext; /* This node's CIphone as viewed externally (context) */ - uint8 ppos; /* Phoneme position in pronunciation */ - uint8 leaf; /* Whether this is a leaf node */ - - /* HMM-state-level stuff here */ - hmm_context_t *ctx; - hmm_t hmm; -} fsg_pnode_t; - -/* Access macros */ -#define fsg_pnode_leaf(p) ((p)->leaf) -#define fsg_pnode_logs2prob(p) ((p)->logs2prob) -#define fsg_pnode_succ(p) ((p)->next.succ) -#define fsg_pnode_fsglink(p) ((p)->next.fsglink) -#define fsg_pnode_sibling(p) ((p)->sibling) -#define fsg_pnode_hmmptr(p) (&((p)->hmm)) -#define fsg_pnode_ci_ext(p) ((p)->ci_ext) -#define fsg_pnode_ppos(p) ((p)->ppos) -#define fsg_pnode_leaf(p) ((p)->leaf) -#define fsg_pnode_ctxt(p) ((p)->ctxt) - -#define fsg_pnode_add_ctxt(p,c) ((p)->ctxt.bv[(c)>>5] |= (1 << ((c)&0x001f))) - -/* - * The following is macroized because its called very frequently - * ::: uint32 fsg_pnode_ctxt_sub (fsg_pnode_ctxt_t *src, fsg_pnode_ctxt_t *sub); - */ -/* - * Subtract bitvector sub from bitvector src (src updated with the result). - * Return 0 if result is all 0, non-zero otherwise. - */ - -#if (FSG_PNODE_CTXT_BVSZ == 1) - #define FSG_PNODE_CTXT_SUB(src,sub) \ - ((src)->bv[0] = (~((sub)->bv[0]) & (src)->bv[0])) -#elif (FSG_PNODE_CTXT_BVSZ == 2) - #define FSG_PNODE_CTXT_SUB(src,sub) \ - (((src)->bv[0] = (~((sub)->bv[0]) & (src)->bv[0])) | \ - ((src)->bv[1] = (~((sub)->bv[1]) & (src)->bv[1]))) -#elif (FSG_PNODE_CTXT_BVSZ == 4) - #define FSG_PNODE_CTXT_SUB(src,sub) \ - (((src)->bv[0] = (~((sub)->bv[0]) & (src)->bv[0])) | \ - ((src)->bv[1] = (~((sub)->bv[1]) & (src)->bv[1])) | \ - ((src)->bv[2] = (~((sub)->bv[2]) & (src)->bv[2])) | \ - ((src)->bv[3] = (~((sub)->bv[3]) & (src)->bv[3]))) -#else - #define FSG_PNODE_CTXT_SUB(src,sub) fsg_pnode_ctxt_sub_generic((src),(sub)) -#endif - -/** - * Collection of lextrees for an FSG. - */ -typedef struct fsg_lextree_s { - fsg_model_t *fsg; /**< The fsg for which this lextree is built. */ - hmm_context_t *ctx; /**< HMM context structure. */ - dict_t *dict; /**< Pronunciation dictionary for this FSG. */ - dict2pid_t *d2p; /**< Context-dependent phone mappings for this FSG. */ - bin_mdef_t *mdef; /**< Model definition (triphone mappings). */ - - /* - * Left and right CIphone sets for each state. - * Left context CIphones for a state S: If word W transitions into S, W's - * final CIphone is in S's {lc}. Words transitioning out of S must consider - * these left context CIphones. - * Similarly, right contexts for state S: If word W transitions out of S, - * W's first CIphone is in S's {rc}. Words transitioning into S must consider - * these right contexts. - * - * NOTE: Words may transition into and out of S INDIRECTLY, with intermediate - * null transitions. - * NOTE: Single-phone words are difficult; only SILENCE right context is - * modelled for them. - * NOTE: Non-silence filler phones aren't included in these sets. Filler - * words don't use context, and present the SILENCE phone as context to - * adjacent words. - */ - int16 **lc; /**< Left context triphone mappings for FSG. */ - int16 **rc; /**< Right context triphone mappings for FSG. */ - - fsg_pnode_t **root; /* root[s] = lextree representing all transitions - out of state s. Note that the "tree" for each - state is actually a collection of trees, linked - via fsg_pnode_t.sibling (root[s]->sibling) */ - fsg_pnode_t **alloc_head; /* alloc_head[s] = head of linear list of all - pnodes allocated for state s */ - int32 n_pnode; /* #HMM nodes in search structure */ - int32 wip; - int32 pip; -} fsg_lextree_t; - -/* Access macros */ -#define fsg_lextree_root(lt,s) ((lt)->root[s]) -#define fsg_lextree_n_pnode(lt) ((lt)->n_pnode) - -/** - * Create, initialize, and return a new phonetic lextree for the given FSG. - */ -fsg_lextree_t *fsg_lextree_init(fsg_model_t *fsg, dict_t *dict, - dict2pid_t *d2p, - bin_mdef_t *mdef, hmm_context_t *ctx, - int32 wip, int32 pip); - -/** - * Free lextrees for an FSG. - */ -void fsg_lextree_free(fsg_lextree_t *fsg); - -/** - * Print an FSG lextree to a file for debugging. - */ -void fsg_lextree_dump(fsg_lextree_t *fsg, FILE *fh); - -/** - * Mark the given pnode as inactive (for search). - */ -void fsg_psubtree_pnode_deactivate(fsg_pnode_t *pnode); - -/** - * Set all flags on in the given context bitvector. - */ -void fsg_pnode_add_all_ctxt(fsg_pnode_ctxt_t *ctxt); - -/** - * Generic variant for arbitrary size - */ -uint32 fsg_pnode_ctxt_sub_generic(fsg_pnode_ctxt_t *src, fsg_pnode_ctxt_t *sub); - -#endif diff --git a/include/fsg_search_internal.h b/include/fsg_search_internal.h deleted file mode 100644 index 7f31359..0000000 --- a/include/fsg_search_internal.h +++ /dev/null @@ -1,153 +0,0 @@ -/* -*- c-basic-offset:4; indent-tabs-mode: nil -*- */ -/* ==================================================================== - * Copyright (c) 1999-2004 Carnegie Mellon University. All rights - * reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * - * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND - * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY - * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * ==================================================================== - * - */ - -/* - * fsg_search_internal.h -- Search structures for FSG decoding. - */ - - -#ifndef __S2_FSG_SEARCH_H__ -#define __S2_FSG_SEARCH_H__ - - -/* SphinxBase headers. */ -#include -#include -#include - -/* Local headers. */ -#include "pocketsphinx_internal.h" -#include "hmm.h" -#include "fsg_history.h" -#include "fsg_lextree.h" - -/** - * Segmentation "iterator" for FSG history. - */ -typedef struct fsg_seg_s { - ps_seg_t base; /**< Base structure. */ - fsg_hist_entry_t **hist; /**< Sequence of history entries. */ - int16 n_hist; /**< Number of history entries. */ - int16 cur; /**< Current position in hist. */ -} fsg_seg_t; - -/** - * Implementation of FSG search (and "FSG set") structure. - */ -typedef struct fsg_search_s { - ps_search_t base; - - hmm_context_t *hmmctx; /**< HMM context. */ - - fsg_model_t *fsg; /**< FSG model */ - struct fsg_lextree_s *lextree;/**< Lextree structure for the currently - active FSG */ - struct fsg_history_s *history;/**< For storing the Viterbi search history */ - - glist_t pnode_active; /**< Those active in this frame */ - glist_t pnode_active_next; /**< Those activated for the next frame */ - - int32 beam_orig; /**< Global pruning threshold */ - int32 pbeam_orig; /**< Pruning threshold for phone transition */ - int32 wbeam_orig; /**< Pruning threshold for word exit */ - float32 beam_factor; /**< Dynamic/adaptive factor (<=1) applied to above - beams to determine actual effective beams. - For implementing absolute pruning. */ - int32 beam, pbeam, wbeam; /**< Effective beams after applying beam_factor */ - int32 lw, pip, wip; /**< Language weights */ - - frame_idx_t frame; /**< Current frame. */ - uint8 final; /**< Decoding is finished for this utterance. */ - uint8 bestpath; /**< Whether to run bestpath search - and confidence annotation at end. */ - float32 ascale; /**< Acoustic score scale for posterior probabilities. */ - - int32 bestscore; /**< For beam pruning */ - int32 bpidx_start; /**< First history entry index this frame */ - - int32 ascr, lscr; /**< Total acoustic and lm score for utt */ - - int32 n_hmm_eval; /**< Total HMMs evaluated this utt */ - int32 n_sen_eval; /**< Total senones evaluated this utt */ - - ptmr_t perf; /**< Performance counter */ - int32 n_tot_frame; - -} fsg_search_t; - -/* Access macros */ -#define fsg_search_frame(s) ((s)->frame) - -/** - * Create, initialize and return a search module. - */ -ps_search_t *fsg_search_init(const char *name, - fsg_model_t *fsg, - cmd_ln_t *config, - acmod_t *acmod, - dict_t *dict, - dict2pid_t *d2p); - -/** - * Deallocate search structure. - */ -void fsg_search_free(ps_search_t *search); - -/** - * Update FSG search module for new or updated FSGs. - */ -int fsg_search_reinit(ps_search_t *fsgs, dict_t *dict, dict2pid_t *d2p); - -/** - * Prepare the FSG search structure for beginning decoding of the next - * utterance. - */ -int fsg_search_start(ps_search_t *search); - -/** - * Step one frame forward through the Viterbi search. - */ -int fsg_search_step(ps_search_t *search, int frame_idx); - -/** - * Windup and clean the FSG search structure after utterance. - */ -int fsg_search_finish(ps_search_t *search); - -/** - * Get hypothesis string from the FSG search. - */ -char const *fsg_search_hyp(ps_search_t *search, int32 *out_score); - -#endif diff --git a/include/kws_detections.h b/include/kws_detections.h deleted file mode 100644 index 855c3bc..0000000 --- a/include/kws_detections.h +++ /dev/null @@ -1,76 +0,0 @@ -/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ -/* ==================================================================== - * Copyright (c) 2014 Carnegie Mellon University. All rights - * reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * - * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND - * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY - * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * ==================================================================== - * - */ - -/* - * kws_detections.h -- Structures for storing keyphrase spotting results. - */ - -#ifndef __KWS_DETECTIONS_H__ -#define __KWS_DETECTIONS_H__ - -/* SphinxBase headers. */ -#include - -/* Local headers. */ -#include "pocketsphinx_internal.h" -#include "hmm.h" - -typedef struct kws_detection_s { - const char* keyphrase; - frame_idx_t sf; - frame_idx_t ef; - int32 prob; - int32 ascr; -} kws_detection_t; - -typedef struct kws_detections_s { - glist_t detect_list; -} kws_detections_t; - -/** - * Reset history structure. - */ -void kws_detections_reset(kws_detections_t *detections); - -/** - * Add history entry. - */ -void kws_detections_add(kws_detections_t *detections, const char* keyphrase, int sf, int ef, int prob, int ascr); - -/** - * Compose hypothesis. - */ -char* kws_detections_hyp_str(kws_detections_t *detections, int frame, int delay); - -#endif /* __KWS_DETECTIONS_H__ */ diff --git a/include/kws_search.h b/include/kws_search.h deleted file mode 100644 index c820afb..0000000 --- a/include/kws_search.h +++ /dev/null @@ -1,142 +0,0 @@ -/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ -/* ==================================================================== - * Copyright (c) 2013 Carnegie Mellon University. All rights - * reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * - * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND - * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY - * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * ==================================================================== - * - */ - -/* - * kws_search.h -- Search structures for keyphrase spotting. - */ - -#ifndef __KWS_SEARCH_H__ -#define __KWS_SEARCH_H__ - -/* SphinxBase headers. */ -#include -#include - -/* Local headers. */ -#include "pocketsphinx_internal.h" -#include "kws_detections.h" -#include "hmm.h" - -/** - * Segmentation "iterator" for KWS history. - */ -typedef struct kws_seg_s { - ps_seg_t base; /**< Base structure. */ - gnode_t *detection; /**< Keyphrase detection correspondent to segment. */ - frame_idx_t last_frame; /**< Last frame to raise the detection */ -} kws_seg_t; - -typedef struct kws_keyphrase_s { - char* word; - int32 threshold; - hmm_t* hmms; - int32 n_hmms; -} kws_keyphrase_t; - -/** - * Implementation of KWS search structure. - */ -typedef struct kws_search_s { - ps_search_t base; - - hmm_context_t *hmmctx; /**< HMM context. */ - - glist_t keyphrases; /**< Keyphrases to spot */ - - kws_detections_t *detections; /**< Keyword spotting history */ - frame_idx_t frame; /**< Frame index */ - - int32 beam; - - int32 plp; /**< Phone loop probability */ - int32 bestscore; /**< For beam pruning */ - int32 def_threshold; /**< default threshold for p(hyp)/p(altern) ratio */ - int32 delay; /**< Delay to wait for best detection score */ - - int32 n_pl; /**< Number of CI phones */ - hmm_t *pl_hmms; /**< Phone loop hmms - hmms of CI phones */ - - ptmr_t perf; /**< Performance counter */ - int32 n_tot_frame; - -} kws_search_t; - -/** - * Create, initialize and return a search module. Gets keyphrases either - * from keyphrase or from a keyphrase file. - */ -ps_search_t *kws_search_init(const char *name, - const char *keyphrase, - const char *keyfile, - cmd_ln_t * config, - acmod_t * acmod, - dict_t * dict, dict2pid_t * d2p); - -/** - * Deallocate search structure. - */ -void kws_search_free(ps_search_t * search); - -/** - * Update KWS search module for new key phrase. - */ -int kws_search_reinit(ps_search_t * kwss, dict_t * dict, dict2pid_t * d2p); - -/** - * Prepare the KWS search structure for beginning decoding of the next - * utterance. - */ -int kws_search_start(ps_search_t * search); - -/** - * Step one frame forward through the Viterbi search. - */ -int kws_search_step(ps_search_t * search, int frame_idx); - -/** - * Windup and clean the KWS search structure after utterance. - */ -int kws_search_finish(ps_search_t * search); - -/** - * Get hypothesis string from the KWS search. - */ -char const *kws_search_hyp(ps_search_t * search, int32 * out_score); - -/** - * Get active keyphrases - */ -char* kws_search_get_keyphrases(ps_search_t * search); - -#endif /* __KWS_SEARCH_H__ */ diff --git a/include/ms_gauden.h b/include/ms_gauden.h deleted file mode 100644 index 9c26cef..0000000 --- a/include/ms_gauden.h +++ /dev/null @@ -1,150 +0,0 @@ -/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ -/* ==================================================================== - * Copyright (c) 1999-2004 Carnegie Mellon University. All rights - * reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * This work was supported in part by funding from the Defense Advanced - * Research Projects Agency and the National Science Foundation of the - * United States of America, and the CMU Sphinx Speech Consortium. - * - * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND - * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY - * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * ==================================================================== - * - */ - -#ifndef _LIBFBS_GAUDEN_H_ -#define _LIBFBS_GAUDEN_H_ - -/** \file ms_gauden.h - * \brief (Sphinx 3.0 specific) Gaussian density module. - * - * Gaussian density distribution implementation. There are two major - * difference bettwen ms_gauden and cont_mgau. One is the fact that - * ms_gauden only take cares of the Gaussian computation part where - * cont_mgau actually take care of senone computation as well. The - * other is the fact that ms_gauden is a multi-stream implementation - * of GMM computation. - * - */ - -/* SphinxBase headers. */ -#include -#include -#include - -/* Local headers. */ -#include "vector.h" -#include "pocketsphinx_internal.h" -#include "hmm.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * \struct gauden_dist_t - * \brief Structure to store distance (density) values for a given input observation wrt density values in some given codebook. - */ -typedef struct { - int32 id; /**< Index of codeword (gaussian density) */ - mfcc_t dist; /**< Density value for input observation wrt above codeword; - NOTE: result in logs3 domain, but var_t used for speed */ - -} gauden_dist_t; - -/** - * \struct gauden_t - * \brief Multivariate gaussian mixture density parameters - */ -typedef struct { - mfcc_t ****mean; /**< mean[codebook][feature][codeword] vector */ - mfcc_t ****var; /**< like mean; diagonal covariance vector only */ - mfcc_t ***det; /**< log(determinant) for each variance vector; - actually, log(sqrt(2*pi*det)) */ - logmath_t *lmath; /**< log math computation */ - int32 n_mgau; /**< Number codebooks */ - int32 n_feat; /**< Number feature streams in each codebook */ - int32 n_density; /**< Number gaussian densities in each codebook-feature stream */ - int32 *featlen; /**< feature length for each feature */ -} gauden_t; - - -/** - * Read mixture gaussian codebooks from the given files. Allocate memory space needed - * for them. Apply the specified variance floor value. - * Return value: ptr to the model created; NULL if error. - * (See Sphinx3 model file-format documentation.) - */ -gauden_t * -gauden_init (char const *meanfile,/**< Input: File containing means of mixture gaussians */ - char const *varfile,/**< Input: File containing variances of mixture gaussians */ - float32 varfloor, /**< Input: Floor value to be applied to variances */ - logmath_t *lmath - ); - -/** Release memory allocated by gauden_init. */ -void gauden_free(gauden_t *g); /**< In: The gauden_t to free */ - -/** Transform Gaussians according to an MLLR matrix (or, eventually, more). */ -int32 gauden_mllr_transform(gauden_t *s, ps_mllr_t *mllr, cmd_ln_t *config); - -/** - * Compute gaussian density values for the given input observation vector wrt the - * specified mixture gaussian codebook (which may consist of several feature streams). - * Density values are left UNnormalized. - * @return 0 if successful, -1 otherwise. - */ -int32 -gauden_dist (gauden_t *g, /**< In: handle to entire ensemble of codebooks */ - int mgau, /**< In: codebook for which density values to be evaluated - (g->{mean,var}[mgau]) */ - int n_top, /**< In: Number top densities to be evaluated */ - mfcc_t **obs, /**< In: Observation vector; obs[f] = for feature f */ - gauden_dist_t **out_dist - /**< Out: n_top best codewords and density values, - in worsening order, for each feature stream. - out_dist[f][i] = i-th best density for feature f. - Caller must allocate memory for this output */ - ); - -/** - Dump the definitionn of Gaussian distribution. -*/ -void gauden_dump (const gauden_t *g /**< In: Gaussian distribution g*/ - ); - -/** - Dump the definition of Gaussian distribution of a particular index to the standard output stream -*/ -void gauden_dump_ind (const gauden_t *g, /**< In: Gaussian distribution g*/ - int senidx /**< In: The senone index of the Gaussian */ - ); - -#ifdef __cplusplus -} -#endif - -#endif /* GAUDEN_H */ diff --git a/include/ms_mgau.h b/include/ms_mgau.h deleted file mode 100644 index b018dcc..0000000 --- a/include/ms_mgau.h +++ /dev/null @@ -1,143 +0,0 @@ -/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ -/* ==================================================================== - * Copyright (c) 1999-2004 Carnegie Mellon University. All rights - * reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * This work was supported in part by funding from the Defense Advanced - * Research Projects Agency and the National Science Foundation of the - * United States of America, and the CMU Sphinx Speech Consortium. - * - * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND - * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY - * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * ==================================================================== - * - */ -/* - * ms_mgau.h -- Essentially a wrapper that wrap up gauden and - * senone. It supports multi-stream. - * - * - * ********************************************** - * CMU ARPA Speech Project - * - * Copyright (c) 1997 Carnegie Mellon University. - * ALL RIGHTS RESERVED. - * ********************************************** - * HISTORY - * $Log$ - * Revision 1.1 2006/04/05 20:27:30 dhdfu - * A Great Reorganzation of header files and executables - * - * Revision 1.3 2006/02/22 16:57:15 arthchan2003 - * Fixed minor dox-doc issue - * - * Revision 1.2 2006/02/22 16:56:01 arthchan2003 - * Merged from SPHINX3_5_2_RCI_IRII_BRANCH: Added ms_mgau.[ch] into the trunk. It is a wrapper of ms_gauden and ms_senone - * - * Revision 1.1.2.4 2005/09/25 18:55:19 arthchan2003 - * Added a flag to turn on and off precomputation. - * - * Revision 1.1.2.3 2005/08/03 18:53:44 dhdfu - * Add memory deallocation functions. Also move all the initialization - * of ms_mgau_model_t into ms_mgau_init (duh!), which entails removing it - * from decode_anytopo and friends. - * - * Revision 1.1.2.2 2005/08/02 21:05:38 arthchan2003 - * 1, Added dist and mgau_active as intermediate variable for computation. 2, Added ms_cont_mgau_frame_eval, which is a multi stream version of GMM computation mainly s3.0 family of tools. 3, Fixed dox-doc. - * - * Revision 1.1.2.1 2005/07/20 19:37:09 arthchan2003 - * Added a multi-stream cont_mgau (ms_mgau) which is a wrapper of both gauden and senone. Add ms_mgau_init and model_set_mllr. This allow eliminating 600 lines of code in decode_anytopo/align/allphone. - * - * - * - */ - -/** \file ms_mgau.h - * - * \brief (Sphinx 3.0 specific) A module that wraps up the code of - * gauden and senone because they are closely related. - * - * At the time at Sphinx 3.1 to 3.2, Ravi has decided to rewrite only - * single-stream part of the code into cont_mgau.[ch]. This marks the - * beginning of historical problem of having two sets of Gaussian - * distribution computation routine, one for single-stream and one of - * multi-stream. - * - * In Sphinx 3.5, when we figure out that it is possible to allow both - * 3.0 family of tools and 3.x family of tools to coexist. This - * becomes one problem we found that very hard to reconcile. That is - * why we currently allow two versions of the code in the code - * base. This is likely to change in the future. - */ - - -#ifndef _LIBFBS_MS_CONT_MGAU_H_ -#define _LIBFBS_MS_CONT_MGAU_H_ - -/* SphinxBase headers. */ -#include -#include -#include - -/* Local headers. */ -#include "acmod.h" -#include "bin_mdef.h" -#include "ms_gauden.h" -#include "ms_senone.h" - -/** \struct ms_mgau_t - \brief Multi-stream mixture gaussian. It is not necessary to be continr -*/ - -typedef struct { - ps_mgau_t base; - gauden_t* g; /**< The codebook */ - senone_t* s; /**< The senone */ - int topn; /**< Top-n gaussian will be computed */ - - /**< Intermediate used in computation */ - gauden_dist_t ***dist; - uint8 *mgau_active; - cmd_ln_t *config; -} ms_mgau_model_t; - -#define ms_mgau_gauden(msg) (msg->g) -#define ms_mgau_senone(msg) (msg->s) -#define ms_mgau_topn(msg) (msg->topn) - -ps_mgau_t* ms_mgau_init(acmod_t *acmod, logmath_t *lmath, bin_mdef_t *mdef); -void ms_mgau_free(ps_mgau_t *g); -int32 ms_cont_mgau_frame_eval(ps_mgau_t * msg, - int16 *senscr, - uint8 *senone_active, - int32 n_senone_active, - mfcc_t ** feat, - int32 frame, - int32 compallsen); -int32 ms_mgau_mllr_transform(ps_mgau_t *s, - ps_mllr_t *mllr); - -#endif /* _LIBFBS_MS_CONT_MGAU_H_*/ - diff --git a/include/ms_senone.h b/include/ms_senone.h deleted file mode 100644 index d92b638..0000000 --- a/include/ms_senone.h +++ /dev/null @@ -1,131 +0,0 @@ -/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ -/* ==================================================================== - * Copyright (c) 1999-2004 Carnegie Mellon University. All rights - * reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * This work was supported in part by funding from the Defense Advanced - * Research Projects Agency and the National Science Foundation of the - * United States of America, and the CMU Sphinx Speech Consortium. - * - * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND - * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY - * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * ==================================================================== - * - */ -/* - * senone.h -- Mixture density weights associated with each tied state. - */ - -#ifndef _MS_SENONE_H_ -#define _MS_SENONE_H_ - - -/* SphinxBase headers. */ -#include -#include -#include -#include - -/* Local headers. */ -#include "ms_gauden.h" -#include "bin_mdef.h" - -/** \file ms_senone.h - * \brief (Sphinx 3.0 specific) multiple streams senones. used with ms_gauden.h - * In Sphinx 3.0 family of tools, ms_senone is used to combine the Gaussian scores. - * Its existence is crucial in Sphinx 3.0 because 3.0 supports both SCHMM and CDHMM. - * There are optimization scheme for SCHMM (e.g. compute the top-N Gaussian) that is - * applicable to SCHMM than CDHMM. This is wrapped in senone_eval_all. - */ - -#ifdef __cplusplus -extern "C" { -#endif - -typedef uint8 senprob_t; /**< Senone logs3-probs, truncated to 8 bits */ - -/** - * \struct senone_t - * \brief 8-bit senone PDF structure. - * - * 8-bit senone PDF structure. Senone pdf values are normalized, floored, converted to - * logs3 domain, and finally truncated to 8 bits precision to conserve memory space. - */ -typedef struct { - senprob_t ***pdf; /**< gaussian density mixture weights, organized two possible - ways depending on n_gauden: - if (n_gauden > 1): pdf[sen][feat][codeword]. Not an - efficient representation--memory access-wise--but - evaluating the many codebooks will be more costly. - if (n_gauden == 1): pdf[feat][codeword][sen]. Optimized - for the shared-distribution semi-continuous case. */ - logmath_t *lmath; /**< log math computation */ - uint32 n_sen; /**< Number senones in this set */ - uint32 n_feat; /**< Number feature streams */ - uint32 n_cw; /**< Number codewords per codebook,stream */ - uint32 n_gauden; /**< Number gaussian density codebooks referred to by senones */ - float32 mixwfloor; /**< floor applied to each PDF entry */ - uint32 *mgau; /**< senone-id -> mgau-id mapping for senones in this set */ - int32 *featscr; /**< The feature score for every senone, will be initialized inside senone_eval_all */ - int32 aw; /**< Inverse acoustic weight */ -} senone_t; - - -/** - * Load a set of senones (mixing weights and mixture gaussian codebook mappings) from - * the given files. Normalize weights for each codebook, apply the given floor, convert - * PDF values to logs3 domain and quantize to 8-bits. - * @return pointer to senone structure created. Caller MUST NOT change its contents. - */ -senone_t *senone_init (gauden_t *g, /**< In: codebooks */ - char const *mixwfile, /**< In: mixing weights file */ - char const *mgau_mapfile,/**< In: file specifying mapping from each - senone to mixture gaussian codebook. - If NULL all senones map to codebook 0 */ - float32 mixwfloor, /**< In: Floor value for senone weights */ - logmath_t *lmath, /**< In: log math computation */ - bin_mdef_t *mdef /**< In: model definition */ - ); - -/** Release memory allocated by senone_init. */ -void senone_free(senone_t *s); /**< In: The senone_t to free */ - -/** - * Evaluate the score for the given senone wrt to the given top N gaussian codewords. - * @return senone score (in logs3 domain). - */ -int32 senone_eval (senone_t *s, int id, /**< In: senone for which score desired */ - gauden_dist_t **dist, /**< In: top N codewords and densities for - all features, to be combined into - senone score. IE, dist[f][i] = i-th - best for feaure f */ - int n_top /**< In: Length of dist[f], for each f */ - ); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/include/ngram_search.h b/include/ngram_search.h deleted file mode 100644 index fe0a98d..0000000 --- a/include/ngram_search.h +++ /dev/null @@ -1,434 +0,0 @@ -/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ -/* ==================================================================== - * Copyright (c) 2008 Carnegie Mellon University. All rights - * reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * This work was supported in part by funding from the Defense Advanced - * Research Projects Agency and the National Science Foundation of the - * United States of America, and the CMU Sphinx Speech Consortium. - * - * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND - * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY - * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * ==================================================================== - * - */ - -/** - * @file ngram_search.h N-Gram based multi-pass search ("FBS") - */ - -#ifndef __NGRAM_SEARCH_H__ -#define __NGRAM_SEARCH_H__ - -/* SphinxBase headers. */ -#include -#include -#include -#include -#include - -/* Local headers. */ -#include "pocketsphinx_internal.h" -#include "hmm.h" - -/** - * Lexical tree node data type. - * - * Not the first HMM for words, which multiplex HMMs based on - * different left contexts. This structure is used both in the - * dynamic HMM tree structure and in the per-word last-phone right - * context fanout. - */ -typedef struct chan_s { - hmm_t hmm; /**< Basic HMM structure. This *must* be first in - the structure because chan_t and root_chan_t are - sometimes used interchangeably */ - struct chan_s *next; /**< first descendant of this channel; or, in the - case of the last phone of a word, the next - alternative right context channel */ - struct chan_s *alt; /**< sibling; i.e., next descendant of parent HMM */ - - int32 ciphone; /**< ciphone for this node */ - union { - int32 penult_phn_wid; /**< list of words whose last phone follows this one; - this field indicates the first of the list; the - rest must be built up in a separate array. Used - only within HMM tree. -1 if none */ - int32 rc_id; /**< right-context id for last phone of words */ - } info; -} chan_t; - -/** - * Lexical tree node data type for the first phone (root) of each dynamic HMM tree - * structure. - * - * Each state may have a different parent static HMM. Most fields are - * similar to those in chan_t. - */ -typedef struct root_chan_s { - hmm_t hmm; /**< Basic HMM structure. This *must* be first in - the structure because chan_t and root_chan_t are - sometimes used interchangeably. */ - chan_t *next; /**< first descendant of this channel */ - - int32 penult_phn_wid; - int32 this_phn_wid; /**< list of words consisting of this single phone; - actually the first of the list, like penult_phn_wid; - -1 if none */ - int16 ciphone; /**< first ciphone of this node; all words rooted at this - node begin with this ciphone */ - int16 ci2phone; /**< second ciphone of this node; one root HMM for each - unique right context */ -} root_chan_t; - -/** - * Back pointer table (forward pass lattice; actually a tree) - */ -typedef struct bptbl_s { - frame_idx_t frame; /**< start or end frame */ - uint8 valid; /**< For absolute pruning */ - uint8 refcnt; /**< Reference count (number of successors) */ - int32 wid; /**< Word index */ - int32 bp; /**< Back Pointer */ - int32 score; /**< Score (best among all right contexts) */ - int32 s_idx; /**< Start of BScoreStack for various right contexts*/ - int32 real_wid; /**< wid of this or latest predecessor real word */ - int32 prev_real_wid; /**< wid of second-last real word */ - int16 last_phone; /**< last phone of this word */ - int16 last2_phone; /**< next-to-last phone of this word */ -} bptbl_t; - -/** - * Segmentation "iterator" for backpointer table results. - */ -typedef struct bptbl_seg_s { - ps_seg_t base; /**< Base structure. */ - int32 *bpidx; /**< Sequence of backpointer IDs. */ - int16 n_bpidx; /**< Number of backpointer IDs. */ - int16 cur; /**< Current position in bpidx. */ -} bptbl_seg_t; - -/* - * Candidates words for entering their last phones. Cleared and rebuilt in each - * frame. - * NOTE: candidates can only be multi-phone, real dictionary words. - */ -typedef struct lastphn_cand_s { - int32 wid; - int32 score; - int32 bp; - int32 next; /* next candidate starting at the same frame */ -} lastphn_cand_t; - -/* - * Since the same instance of a word (i.e., ) reaches its last - * phone several times, we can compute its best BP and LM transition score info - * just the first time and cache it for future occurrences. Structure for such - * a cache. - */ -typedef struct { - int32 sf; /* Start frame */ - int32 dscr; /* Delta-score upon entering last phone */ - int32 bp; /* Best BP */ -} last_ltrans_t; - -#define CAND_SF_ALLOCSIZE 32 -typedef struct { - int32 bp_ef; - int32 cand; -} cand_sf_t; - -/* - * Structure for reorganizing the BP table entries in the current frame according - * to distinct right context ci-phones. Each entry contains the best BP entry for - * a given right context. Each successor word will pick up the correct entry based - * on its first ci-phone. - */ -typedef struct bestbp_rc_s { - int32 score; - int32 path; /* BP table index corresponding to this entry */ - int32 lc; /* right most ci-phone of above BP entry word */ -} bestbp_rc_t; - -#define NO_BP -1 - -/** - * Various statistics for profiling. - */ -typedef struct ngram_search_stats_s { - int32 n_phone_eval; - int32 n_root_chan_eval; - int32 n_nonroot_chan_eval; - int32 n_last_chan_eval; - int32 n_word_lastchan_eval; - int32 n_lastphn_cand_utt; - int32 n_fwdflat_chan; - int32 n_fwdflat_words; - int32 n_fwdflat_word_transition; - int32 n_senone_active_utt; -} ngram_search_stats_t; - - -/** - * N-Gram search module structure. - */ -struct ngram_search_s { - ps_search_t base; - ngram_model_t *lmset; /**< Set of language models. */ - hmm_context_t *hmmctx; /**< HMM context. */ - - /* Flags to quickly indicate which passes are enabled. */ - uint8 fwdtree; - uint8 fwdflat; - uint8 bestpath; - - /* State of procesing. */ - uint8 done; - - /* Allocators */ - listelem_alloc_t *chan_alloc; /**< For chan_t */ - listelem_alloc_t *root_chan_alloc; /**< For root_chan_t */ - listelem_alloc_t *latnode_alloc; /**< For latnode_t */ - - /** - * Search structure of HMM instances. - * - * The word triphone sequences (HMM instances) are transformed - * into tree structures, one tree per unique left triphone in the - * entire dictionary (actually diphone, since its left context - * varies dyamically during the search process). The entire set - * of trees of channels is allocated once and for all during - * initialization (since dynamic management of active CHANs is - * time consuming), with one exception: the last phones of words, - * that need multiple right context modelling, are not maintained - * in this static structure since there are too many of them and - * few are active at any time. Instead they are maintained as - * linked lists of CHANs, one list per word, and each CHAN in this - * set is allocated only on demand and freed if inactive. - */ - root_chan_t *root_chan; /**< Roots of search tree. */ - int32 n_root_chan_alloc; /**< Number of root_chan allocated */ - int32 n_root_chan; /**< Number of valid root_chan */ - int32 n_nonroot_chan; /**< Number of valid non-root channels */ - int32 max_nonroot_chan; /**< Maximum possible number of non-root channels */ - root_chan_t *rhmm_1ph; /**< Root HMMs for single-phone words */ - - /** - * Channels associated with a given word (only used for right - * contexts, single-phone words in fwdtree search, and word HMMs - * in fwdflat search). WARNING: For single-phone words and - * fwdflat search, this actually contains pointers to root_chan_t, - * which are allocated using root_chan_alloc. This is a - * suboptimal state of affairs. - */ - chan_t **word_chan; - bitvec_t *word_active; /**< array of active flags for all words. */ - - /** - * Each node in the HMM tree structure may point to a set of words - * whose last phone would follow that node in the tree structure - * (but is not included in the tree structure for reasons - * explained above). The channel node points to one word in this - * set of words. The remaining words are linked through - * homophone_set[]. - * - * Single-phone words are not represented in the HMM tree; they - * are kept in word_chan. - * - * Specifically, homophone_set[w] = wid of next word in the same - * set as w. - */ - int32 *homophone_set; - int32 *single_phone_wid; /**< list of single-phone word ids */ - int32 n_1ph_words; /**< Number single phone words in dict (total) */ - int32 n_1ph_LMwords; /**< Number single phone dict words also in LM; - these come first in single_phone_wid */ - /** - * Array of active channels for current and next frame. - * - * In any frame, only some HMM tree nodes are active. - * active_chan_list[f mod 2] = list of nonroot channels in the HMM - * tree active in frame f. - */ - chan_t ***active_chan_list; - int32 n_active_chan[2]; /**< Number entries in active_chan_list */ - /** - * Array of active multi-phone words for current and next frame. - * - * Similarly to active_chan_list, active_word_list[f mod 2] = list - * of word ids for which active channels exist in word_chan in - * frame f. - * - * Statically allocated single-phone words are always active and - * should not appear in this list. - */ - int32 **active_word_list; - int32 n_active_word[2]; /**< Number entries in active_word_list */ - - /* - * FIXME: Document all of these bits. - */ - lastphn_cand_t *lastphn_cand; - int32 n_lastphn_cand; - last_ltrans_t *last_ltrans; /* one per word */ - int32 cand_sf_alloc; - cand_sf_t *cand_sf; - bestbp_rc_t *bestbp_rc; - - bptbl_t *bp_table; /* Forward pass lattice */ - int32 bpidx; /* First free BPTable entry */ - int32 bp_table_size; - int32 *bscore_stack; /* Score stack for all possible right contexts */ - int32 bss_head; /* First free BScoreStack entry */ - int32 bscore_stack_size; - - int32 n_frame_alloc; /**< Number of frames allocated in bp_table_idx and friends. */ - int32 n_frame; /**< Number of frames actually present. */ - int32 *bp_table_idx; /* First BPTable entry for each frame */ - int32 *word_lat_idx; /* BPTable index for any word in current frame; - cleared before each frame */ - - /* - * Flat lexicon (2nd pass) search stuff. - */ - ps_latnode_t **frm_wordlist; /**< List of active words in each frame. */ - int32 *fwdflat_wordlist; /**< List of active word IDs for utterance. */ - bitvec_t *expand_word_flag; - int32 *expand_word_list; - int32 n_expand_words; - int32 min_ef_width; - int32 max_sf_win; - float32 fwdflat_fwdtree_lw_ratio; - - int32 best_score; /**< Best Viterbi path score. */ - int32 last_phone_best_score; /**< Best Viterbi path score for last phone. */ - int32 renormalized; - - /* - * DAG (3rd pass) search stuff. - */ - float32 bestpath_fwdtree_lw_ratio; - float32 ascale; /**< Acoustic score scale for posterior probabilities. */ - - ngram_search_stats_t st; /**< Various statistics for profiling. */ - ptmr_t fwdtree_perf; - ptmr_t fwdflat_perf; - ptmr_t bestpath_perf; - int32 n_tot_frame; - - /* A collection of beam widths. */ - int32 beam; - int32 dynamic_beam; - int32 pbeam; - int32 wbeam; - int32 lpbeam; - int32 lponlybeam; - int32 fwdflatbeam; - int32 fwdflatwbeam; - int32 fillpen; - int32 silpen; - int32 wip; - int32 nwpen; - int32 pip; - int32 maxwpf; - int32 maxhmmpf; -}; -typedef struct ngram_search_s ngram_search_t; - -/** - * Initialize the N-Gram search module. - */ -ps_search_t *ngram_search_init(const char *name, - ngram_model_t *lm, - cmd_ln_t *config, - acmod_t *acmod, - dict_t *dict, - dict2pid_t *d2p); - -/** - * Finalize the N-Gram search module. - */ -void ngram_search_free(ps_search_t *ngs); - -/** - * Record the current frame's index in the backpointer table. - * - * @return the current backpointer index. - */ -int ngram_search_mark_bptable(ngram_search_t *ngs, int frame_idx); - -/** - * Enter a word in the backpointer table. - */ -void ngram_search_save_bp(ngram_search_t *ngs, int frame_idx, int32 w, - int32 score, int32 path, int32 rc); - -/** - * Allocate last phone channels for all possible right contexts for word w. - */ -void ngram_search_alloc_all_rc(ngram_search_t *ngs, int32 w); - -/** - * Allocate last phone channels for all possible right contexts for word w. - */ -void ngram_search_free_all_rc(ngram_search_t *ngs, int32 w); - -/** - * Find the best word exit for the current frame in the backpointer table. - * - * @return the backpointer index of the best word exit. - */ -int ngram_search_find_exit(ngram_search_t *ngs, int frame_idx, int32 *out_best_score); - -/** - * Backtrace from a given backpointer index to obtain a word hypothesis. - * - * @return a read-only string with the best hypothesis. - */ -char const *ngram_search_bp_hyp(ngram_search_t *ngs, int bpidx); - -/** - * Compute language and acoustic scores for backpointer table entries. - */ -void ngram_compute_seg_scores(ngram_search_t *ngs, float32 lwf); - -/** - * Construct a word lattice from the current hypothesis. - */ -ps_lattice_t *ngram_search_lattice(ps_search_t *search); - -/** - * Get the exit score for a backpointer entry with a given right context. - */ -int32 ngram_search_exit_score(ngram_search_t *ngs, bptbl_t *pbe, int rcphone); - -/** - * Sets the global language model. - * - * Sets the language model to use if nothing was passed in configuration - */ -void ngram_search_set_lm(ngram_model_t *lm); - -#endif /* __NGRAM_SEARCH_H__ */ diff --git a/include/ngram_search_fwdflat.h b/include/ngram_search_fwdflat.h deleted file mode 100644 index 026b397..0000000 --- a/include/ngram_search_fwdflat.h +++ /dev/null @@ -1,81 +0,0 @@ -/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ -/* ==================================================================== - * Copyright (c) 2008 Carnegie Mellon University. All rights - * reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * This work was supported in part by funding from the Defense Advanced - * Research Projects Agency and the National Science Foundation of the - * United States of America, and the CMU Sphinx Speech Consortium. - * - * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND - * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY - * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * ==================================================================== - * - */ - -/** - * @file ngram_search_fwdflat.h Flat lexicon based Viterbi search. - */ - -#ifndef __NGRAM_SEARCH_FWDFLAT_H__ -#define __NGRAM_SEARCH_FWDFLAT_H__ - -/* SphinxBase headers. */ - -/* Local headers. */ -#include "ngram_search.h" - -/** - * Initialize N-Gram search for fwdflat decoding. - */ -void ngram_fwdflat_init(ngram_search_t *ngs); - -/** - * Release memory associated with fwdflat decoding. - */ -void ngram_fwdflat_deinit(ngram_search_t *ngs); - -/** - * Rebuild search structures for updated language models. - */ -int ngram_fwdflat_reinit(ngram_search_t *ngs); - -/** - * Start fwdflat decoding for an utterance. - */ -void ngram_fwdflat_start(ngram_search_t *ngs); - -/** - * Search one frame forward in an utterance. - */ -int ngram_fwdflat_search(ngram_search_t *ngs, int frame_idx); - -/** - * Finish fwdflat decoding for an utterance. - */ -void ngram_fwdflat_finish(ngram_search_t *ngs); - - -#endif /* __NGRAM_SEARCH_FWDFLAT_H__ */ diff --git a/include/ngram_search_fwdtree.h b/include/ngram_search_fwdtree.h deleted file mode 100644 index 8063ab7..0000000 --- a/include/ngram_search_fwdtree.h +++ /dev/null @@ -1,83 +0,0 @@ -/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ -/* ==================================================================== - * Copyright (c) 2008 Carnegie Mellon University. All rights - * reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * This work was supported in part by funding from the Defense Advanced - * Research Projects Agency and the National Science Foundation of the - * United States of America, and the CMU Sphinx Speech Consortium. - * - * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND - * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY - * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * ==================================================================== - * - */ - -/** - * @file ngram_search_fwdtree.h Lexicon tree based Viterbi search. - */ - -#ifndef __NGRAM_SEARCH_FWDTREE_H__ -#define __NGRAM_SEARCH_FWDTREE_H__ - -/* SphinxBase headers. */ - -/* Local headers. */ -#include "ngram_search.h" - -/** - * Initialize N-Gram search for fwdtree decoding. - */ -void ngram_fwdtree_init(ngram_search_t *ngs); - -/** - * Release memory associated with fwdtree decoding. - */ -void ngram_fwdtree_deinit(ngram_search_t *ngs); - -/** - * Rebuild search structures for updated language models. - */ -int ngram_fwdtree_reinit(ngram_search_t *ngs); - -/** - * Start fwdtree decoding for an utterance. - */ -void ngram_fwdtree_start(ngram_search_t *ngs); - -/** - * Search one frame forward in an utterance. - * - * @return Number of frames searched (either 0 or 1). - */ -int ngram_fwdtree_search(ngram_search_t *ngs, int frame_idx); - -/** - * Finish fwdtree decoding for an utterance. - */ -void ngram_fwdtree_finish(ngram_search_t *ngs); - - -#endif /* __NGRAM_SEARCH_FWDTREE_H__ */ diff --git a/include/phone_loop_search.h b/include/phone_loop_search.h deleted file mode 100644 index db776e9..0000000 --- a/include/phone_loop_search.h +++ /dev/null @@ -1,102 +0,0 @@ -/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ -/* ==================================================================== - * Copyright (c) 2008 Carnegie Mellon University. All rights - * reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * This work was supported in part by funding from the Defense Advanced - * Research Projects Agency and the National Science Foundation of the - * United States of America, and the CMU Sphinx Speech Consortium. - * - * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND - * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY - * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * ==================================================================== - * - */ - -/** - * @file phone_loop_search.h Fast and rough context-independent - * phoneme loop search. - * - * This exists for the purposes of phoneme lookahead, and thus it - * actually does not do phoneme recognition (it wouldn't be very - * accurate anyway). - */ - -#ifndef __PHONE_LOOP_SEARCH_H__ -#define __PHONE_LOOP_SEARCH_H__ - -/* SphinxBase headers. */ -#include -#include -#include -#include - -/* Local headers. */ -#include "pocketsphinx_internal.h" -#include "hmm.h" - -/** - * Renormalization event. - */ -struct phone_loop_renorm_s { - int frame_idx; /**< Frame of renormalization. */ - int32 norm; /**< Normalization constant. */ -}; -typedef struct phone_loop_renorm_s phone_loop_renorm_t; - -/** - * Phone loop search structure. - */ -struct phone_loop_search_s { - ps_search_t base; /**< Base search structure. */ - hmm_t *hmms; /**< Basic HMM structures for CI phones. */ - hmm_context_t *hmmctx; /**< HMM context structure. */ - int16 frame; /**< Current frame being searched. */ - int16 n_phones; /**< Size of phone array. */ - int32 **pen_buf; /**< Penalty buffer */ - int16 pen_buf_ptr; /**< Pointer for frame to fill in penalty buffer */ - int32 *penalties; /**< Penalties for CI phones in current frame */ - float64 penalty_weight; /**< Weighting factor for penalties */ - - int32 best_score; /**< Best Viterbi score in current frame. */ - int32 beam; /**< HMM pruning beam width. */ - int32 pbeam; /**< Phone exit pruning beam width. */ - int32 pip; /**< Phone insertion penalty ("language score"). */ - int window; /**< Window size for phoneme lookahead */ - glist_t renorm; /**< List of renormalizations. */ -}; -typedef struct phone_loop_search_s phone_loop_search_t; - -ps_search_t *phone_loop_search_init(cmd_ln_t *config, - acmod_t *acmod, - dict_t *dict); - -/** - * Return lookahead heuristic score for a specific phone. - */ -#define phone_loop_search_score(pls,ci) \ - ((pls == NULL) ? 0 : (pls->penalties[ci])) - -#endif /* __PHONE_LOOP_SEARCH_H__ */ diff --git a/include/ps_lattice_internal.h b/include/ps_lattice_internal.h deleted file mode 100644 index 4e5f7dd..0000000 --- a/include/ps_lattice_internal.h +++ /dev/null @@ -1,282 +0,0 @@ -/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ -/* ==================================================================== - * Copyright (c) 2008 Carnegie Mellon University. All rights - * reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * This work was supported in part by funding from the Defense Advanced - * Research Projects Agency and the National Science Foundation of the - * United States of America, and the CMU Sphinx Speech Consortium. - * - * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND - * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY - * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * ==================================================================== - * - */ - -/** - * @file ps_lattice_internal.h Word graph search implementation - */ - -#ifndef __PS_LATTICE_INTERNAL_H__ -#define __PS_LATTICE_INTERNAL_H__ - -/** - * Linked list of DAG link pointers. - * - * Because the same link structure is used for forward and reverse - * links, as well as for the agenda used in bestpath search, we can't - * store the list pointer inside latlink_t. We could use glist_t - * here, but it wastes 4 bytes per entry on 32-bit machines. - */ -typedef struct latlink_list_s { - ps_latlink_t *link; - struct latlink_list_s *next; -} latlink_list_t; - -/** - * Word graph structure used in bestpath/nbest search. - */ -struct ps_lattice_s { - int refcount; /**< Reference count. */ - - logmath_t *lmath; /**< Log-math object. */ - ps_search_t *search; /**< Search (if generated by search). */ - dict_t *dict; /**< Dictionary for this DAG. */ - int32 silence; /**< Silence word ID. */ - int32 frate; /**< Frame rate. */ - - ps_latnode_t *nodes; /**< List of all nodes. */ - ps_latnode_t *start; /**< Starting node. */ - ps_latnode_t *end; /**< Ending node. */ - - frame_idx_t n_frames; /**< Number of frames for this utterance. */ - int32 n_nodes; /**< Number of nodes in this lattice. */ - int32 final_node_ascr; /**< Acoustic score of implicit link exiting final node. */ - int32 norm; /**< Normalizer for posterior probabilities. */ - char *hyp_str; /**< Current hypothesis string. */ - - listelem_alloc_t *latnode_alloc; /**< Node allocator for this DAG. */ - listelem_alloc_t *latlink_alloc; /**< Link allocator for this DAG. */ - listelem_alloc_t *latlink_list_alloc; /**< List element allocator for this DAG. */ - - /* This will probably be replaced with a heap. */ - latlink_list_t *q_head; /**< Queue of links for traversal. */ - latlink_list_t *q_tail; /**< Queue of links for traversal. */ -}; - -/** - * Links between DAG nodes. - * - * A link corresponds to a single hypothesized instance of a word with - * a given start and end point. - - */ -struct ps_latlink_s { - struct ps_latnode_s *from; /**< From node */ - struct ps_latnode_s *to; /**< To node */ - struct ps_latlink_s *best_prev; - int32 ascr; /**< Score for from->wid (from->sf to this->ef) */ - int32 path_scr; /**< Best path score from root of DAG */ - frame_idx_t ef; /**< Ending frame of this word */ - int32 alpha; /**< Forward probability of this link P(w,o_1^{ef}) */ - int32 beta; /**< Backward probability of this link P(w|o_{ef+1}^T) */ -}; - -/** - * DAG nodes. - * - * A node corresponds to a number of hypothesized instances of a word - * which all share the same starting point. - */ -struct ps_latnode_s { - int32 id; /**< Unique id for this node */ - int32 wid; /**< Dictionary word id */ - int32 basewid; /**< Dictionary base word id */ - /* FIXME: These are (ab)used to store backpointer indices, therefore they MUST be 32 bits. */ - int32 fef; /**< First end frame */ - int32 lef; /**< Last end frame */ - frame_idx_t sf; /**< Start frame */ - int16 reachable; /**< From \verbatim \endverbatim or \verbatim \endverbatim */ - int32 node_id; /**< Node from fsg model, used to map lattice back to model */ - union { - glist_t velist; /**< List of history entries with different lmstate (tst only) */ - int32 fanin; /**< Number nodes with links to this node */ - int32 rem_score; /**< Estimated best score from node.sf to end */ - int32 best_exit; /**< Best exit score (used for final nodes only) */ - } info; - latlink_list_t *exits; /**< Links out of this node */ - latlink_list_t *entries; /**< Links into this node */ - - struct ps_latnode_s *alt; /**< Node with alternate pronunciation for this word */ - struct ps_latnode_s *next; /**< Next node in DAG (no ordering implied) */ -}; - -/** - * Segmentation "iterator" for backpointer table results. - */ -typedef struct dag_seg_s { - ps_seg_t base; /**< Base structure. */ - ps_latlink_t **links; /**< Array of lattice links. */ - int32 norm; /**< Normalizer for posterior probabilities. */ - int16 n_links; /**< Number of lattice links. */ - int16 cur; /**< Current position in bpidx. */ -} dag_seg_t; - -/** - * Partial path structure used in N-best (A*) search. - * - * Each partial path (latpath_t) is constructed by extending another - * partial path--parent--by one node. - */ -typedef struct ps_latpath_s { - ps_latnode_t *node; /**< Node ending this path. */ - struct ps_latpath_s *parent; /**< Previous element in this path. */ - struct ps_latpath_s *next; /**< Pointer to next path in list of paths. */ - int32 score; /**< Exact score from start node up to node->sf. */ -} ps_latpath_t; - -/** - * A* search structure. - */ -typedef struct ps_astar_s { - ps_lattice_t *dag; - ngram_model_t *lmset; - float32 lwf; - - frame_idx_t sf; - frame_idx_t ef; - int32 w1; - int32 w2; - - int32 n_hyp_tried; - int32 n_hyp_insert; - int32 n_hyp_reject; - int32 insert_depth; - int32 n_path; - - ps_latpath_t *path_list; - ps_latpath_t *path_tail; - ps_latpath_t *top; - - glist_t hyps; /**< List of hypothesis strings. */ - listelem_alloc_t *latpath_alloc; /**< Path allocator for N-best search. */ -} ps_astar_t; - -/** - * Segmentation "iterator" for A* search results. - */ -typedef struct astar_seg_s { - ps_seg_t base; - ps_latnode_t **nodes; - int n_nodes; - int cur; -} astar_seg_t; - -/** - * Construct an empty word graph with reference to a search structure. - */ -ps_lattice_t *ps_lattice_init_search(ps_search_t *search, int n_frame); - -/** - * Insert penalty for fillers - */ -void ps_lattice_penalize_fillers(ps_lattice_t *dag, int32 silpen, int32 fillpen); - -/** - * Remove nodes marked as unreachable. - */ -void ps_lattice_delete_unreachable(ps_lattice_t *dag); - -/** - * Add an edge to the traversal queue. - */ -void ps_lattice_pushq(ps_lattice_t *dag, ps_latlink_t *link); - -/** - * Remove an edge from the traversal queue. - */ -ps_latlink_t *ps_lattice_popq(ps_lattice_t *dag); - -/** - * Clear and reset the traversal queue. - */ -void ps_lattice_delq(ps_lattice_t *dag); - -/** - * Create a new lattice link element. - */ -latlink_list_t *latlink_list_new(ps_lattice_t *dag, ps_latlink_t *link, - latlink_list_t *next); - -/** - * Get hypothesis string after bestpath search. - */ -char const *ps_lattice_hyp(ps_lattice_t *dag, ps_latlink_t *link); - -/** - * Get hypothesis segmentation iterator after bestpath search. - */ -ps_seg_t *ps_lattice_seg_iter(ps_lattice_t *dag, ps_latlink_t *link, - float32 lwf); - -/** - * Begin N-Gram based A* search on a word graph. - * - * @param sf Starting frame for N-best search. - * @param ef Ending frame for N-best search, or -1 for last frame. - * @param w1 First context word, or -1 for none. - * @param w2 Second context word, or -1 for none. - * @return 0 for success, <0 on error. - */ -ps_astar_t *ps_astar_start(ps_lattice_t *dag, - ngram_model_t *lmset, - float32 lwf, - int sf, int ef, - int w1, int w2); - -/** - * Find next best hypothesis of A* on a word graph. - * - * @return a complete path, or NULL if no more hypotheses exist. - */ -ps_latpath_t *ps_astar_next(ps_astar_t *nbest); - -/** - * Finish N-best search, releasing resources associated with it. - */ -void ps_astar_finish(ps_astar_t *nbest); - -/** - * Get hypothesis string from A* search. - */ -char const *ps_astar_hyp(ps_astar_t *nbest, ps_latpath_t *path); - -/** - * Get hypothesis segmentation from A* search. - */ -ps_seg_t *ps_astar_seg_iter(ps_astar_t *astar, ps_latpath_t *path, float32 lwf); - - -#endif /* __PS_LATTICE_INTERNAL_H__ */ diff --git a/include/ptm_mgau.h b/include/ptm_mgau.h deleted file mode 100644 index 0b3ac63..0000000 --- a/include/ptm_mgau.h +++ /dev/null @@ -1,103 +0,0 @@ -/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ -/* ==================================================================== - * Copyright (c) 1999-2010 Carnegie Mellon University. All rights - * reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * This work was supported in part by funding from the Defense Advanced - * Research Projects Agency and the National Science Foundation of the - * United States of America, and the CMU Sphinx Speech Consortium. - * - * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND - * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY - * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * ==================================================================== - * - */ -/** - * @file ptm_mgau.h Fast phonetically-tied mixture evaluation. - * @author David Huggins-Daines - */ - -#ifndef __PTM_MGAU_H__ -#define __PTM_MGAU_H__ - -/* SphinxBase headesr. */ -#include -#include -#include - -/* Local headers. */ -#include "acmod.h" -#include "hmm.h" -#include "bin_mdef.h" -#include "ms_gauden.h" - -typedef struct ptm_mgau_s ptm_mgau_t; - -typedef struct ptm_topn_s { - int32 cw; /**< Codeword index. */ - int32 score; /**< Score. */ -} ptm_topn_t; - -typedef struct ptm_fast_eval_s { - ptm_topn_t ***topn; /**< Top-N for each codebook (mgau x feature x topn) */ - bitvec_t *mgau_active; /**< Set of active codebooks */ -} ptm_fast_eval_t; - -struct ptm_mgau_s { - ps_mgau_t base; /**< base structure. */ - cmd_ln_t *config; /**< Configuration parameters */ - gauden_t *g; /**< Set of Gaussians. */ - int32 n_sen; /**< Number of senones. */ - uint8 *sen2cb; /**< Senone to codebook mapping. */ - uint8 ***mixw; /**< Mixture weight distributions by feature, codeword, senone */ - mmio_file_t *sendump_mmap;/* Memory map for mixw (or NULL if not mmap) */ - uint8 *mixw_cb; /* Mixture weight codebook, if any (assume it contains 16 values) */ - int16 max_topn; - int16 ds_ratio; - - ptm_fast_eval_t *hist; /**< Fast evaluation info for past frames. */ - ptm_fast_eval_t *f; /**< Fast eval info for current frame. */ - int n_fast_hist; /**< Number of past frames tracked. */ - - /* Log-add table for compressed values. */ - logmath_t *lmath_8b; - /* Log-add object for reloading means/variances. */ - logmath_t *lmath; -}; - -ps_mgau_t *ptm_mgau_init(acmod_t *acmod, bin_mdef_t *mdef); -void ptm_mgau_free(ps_mgau_t *s); -int ptm_mgau_frame_eval(ps_mgau_t *s, - int16 *senone_scores, - uint8 *senone_active, - int32 n_senone_active, - mfcc_t **featbuf, - int32 frame, - int32 compallsen); -int ptm_mgau_mllr_transform(ps_mgau_t *s, - ps_mllr_t *mllr); - - -#endif /* __PTM_MGAU_H__ */ diff --git a/include/s2_semi_mgau.h b/include/s2_semi_mgau.h deleted file mode 100644 index f127b5d..0000000 --- a/include/s2_semi_mgau.h +++ /dev/null @@ -1,98 +0,0 @@ -/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ -/* ==================================================================== - * Copyright (c) 1999-2004 Carnegie Mellon University. All rights - * reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * This work was supported in part by funding from the Defense Advanced - * Research Projects Agency and the National Science Foundation of the - * United States of America, and the CMU Sphinx Speech Consortium. - * - * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND - * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY - * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * ==================================================================== - * - */ -/* - * Interface for "semi-continuous vector quantization", a.k.a. Sphinx2 - * fast GMM computation. - */ - -#ifndef __S2_SEMI_MGAU_H__ -#define __S2_SEMI_MGAU_H__ - -/* SphinxBase headesr. */ -#include -#include -#include - -/* Local headers. */ -#include "acmod.h" -#include "hmm.h" -#include "bin_mdef.h" -#include "ms_gauden.h" - -typedef struct vqFeature_s vqFeature_t; - -typedef struct s2_semi_mgau_s s2_semi_mgau_t; -struct s2_semi_mgau_s { - ps_mgau_t base; /**< base structure. */ - cmd_ln_t *config; /* configuration parameters */ - - gauden_t *g; /* Set of Gaussians (pointers below point in here and will go away soon) */ - - uint8 ***mixw; /* mixture weight distributions */ - mmio_file_t *sendump_mmap;/* memory map for mixw (or NULL if not mmap) */ - - uint8 *mixw_cb; /* mixture weight codebook, if any (assume it contains 16 values) */ - int32 n_sen; /* Number of senones */ - uint8 *topn_beam; /* Beam for determining per-frame top-N densities */ - int16 max_topn; - int16 ds_ratio; - - vqFeature_t ***topn_hist; /**< Top-N scores and codewords for past frames. */ - uint8 **topn_hist_n; /**< Variable top-N for past frames. */ - vqFeature_t **f; /**< Topn-N for currently scoring frame. */ - int n_topn_hist; /**< Number of past frames tracked. */ - - /* Log-add table for compressed values. */ - logmath_t *lmath_8b; - /* Log-add object for reloading means/variances. */ - logmath_t *lmath; -}; - -ps_mgau_t *s2_semi_mgau_init(acmod_t *acmod); -void s2_semi_mgau_free(ps_mgau_t *s); -int s2_semi_mgau_frame_eval(ps_mgau_t *s, - int16 *senone_scores, - uint8 *senone_active, - int32 n_senone_active, - mfcc_t **featbuf, - int32 frame, - int32 compallsen); -int s2_semi_mgau_mllr_transform(ps_mgau_t *s, - ps_mllr_t *mllr); - - -#endif /* __S2_SEMI_MGAU_H__ */ diff --git a/include/tied_mgau_common.h b/include/tied_mgau_common.h deleted file mode 100644 index c8c5320..0000000 --- a/include/tied_mgau_common.h +++ /dev/null @@ -1,121 +0,0 @@ -/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */ -/* ==================================================================== - * Copyright (c) 1999-2010 Carnegie Mellon University. All rights - * reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * This work was supported in part by funding from the Defense Advanced - * Research Projects Agency and the National Science Foundation of the - * United States of America, and the CMU Sphinx Speech Consortium. - * - * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND - * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY - * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * ==================================================================== - * - */ - -/** - * @file tied_mgau_common.h - * @brief Common code shared between SC and PTM (tied-state) models. - */ - -#ifndef __TIED_MGAU_COMMON_H__ -#define __TIED_MGAU_COMMON_H__ - -#include -#include - -#define MGAU_MIXW_VERSION "1.0" /* Sphinx-3 file format version for mixw */ -#define MGAU_PARAM_VERSION "1.0" /* Sphinx-3 file format version for mean/var */ -#define NONE -1 -#define WORST_DIST (int32)(0x80000000) - -/** Subtract GMM component b (assumed to be positive) and saturate */ -#ifdef FIXED_POINT -#define GMMSUB(a,b) \ - (((a)-(b) > a) ? (INT_MIN) : ((a)-(b))) -/** Add GMM component b (assumed to be positive) and saturate */ -#define GMMADD(a,b) \ - (((a)+(b) < a) ? (INT_MAX) : ((a)+(b))) -#else -#define GMMSUB(a,b) ((a)-(b)) -#define GMMADD(a,b) ((a)+(b)) -#endif - -#ifndef MIN -#define MIN(a,b) ((a) < (b) ? (a) : (b)) -#endif - - -#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) -#define LOGMATH_INLINE static inline -#elif defined(_MSC_VER) -#define LOGMATH_INLINE __inline -#else -#define LOGMATH_INLINE static -#endif - -/* Allocate 0..159 for negated quantized mixture weights and 0..96 for - * negated normalized acoustic scores, so that the combination of the - * two (for a single mixture) can never exceed 255. */ -#define MAX_NEG_MIXW 159 /**< Maximum negated mixture weight value. */ -#define MAX_NEG_ASCR 96 /**< Maximum negated acoustic score value. */ - -/** - * Quickly log-add two negated log probabilities. - * - * @param lmath The log-math object - * @param mlx A negative log probability (0 < mlx < 255) - * @param mly A negative log probability (0 < mly < 255) - * @return -log(exp(-mlx)+exp(-mly)) - * - * We can do some extra-fast log addition since we know that - * mixw+ascr is always less than 256 and hence x-y is also always less - * than 256. This relies on some cooperation from logmath_t which - * will never produce a logmath table smaller than 256 entries. - * - * Note that the parameters are *negated* log probabilities (and - * hence, are positive numbers), as is the return value. This is the - * key to the "fastness" of this function. - */ -LOGMATH_INLINE int -fast_logmath_add(logmath_t *lmath, int mlx, int mly) -{ - logadd_t *t = LOGMATH_TABLE(lmath); - int d, r; - - /* d must be positive, obviously. */ - if (mlx > mly) { - d = (mlx - mly); - r = mly; - } - else { - d = (mly - mlx); - r = mlx; - } - - return r - (((uint8 *)t->table)[d]); -} - -#endif /* __TIED_MGAU_COMMON_H__ */ diff --git a/include/vector.h b/include/vector.h deleted file mode 100644 index ed81398..0000000 --- a/include/vector.h +++ /dev/null @@ -1,89 +0,0 @@ -/* ==================================================================== - * Copyright (c) 1999-2004 Carnegie Mellon University. All rights - * reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * This work was supported in part by funding from the Defense Advanced - * Research Projects Agency and the National Science Foundation of the - * United States of America, and the CMU Sphinx Speech Consortium. - * - * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND - * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY - * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * ==================================================================== - * - */ - -/* - * vector.h -- vector routines. - * - * ********************************************** - * CMU ARPA Speech Project - * - * Copyright (c) 1997 Carnegie Mellon University. - * ALL RIGHTS RESERVED. - * ********************************************** - */ - - -#ifndef __VECTOR_H__ -#define __VECTOR_H__ - -/* System headers. */ -#include - -/* SphinxBase headers. */ -#include - -typedef float32 *vector_t; - -/* - * The reason for some of the "trivial" routines below is that they could be OPTIMIZED for SPEED - * at some point. - */ - - -/* Floor all elements of v[0..dim-1] to min value of f */ -void vector_floor(vector_t v, int32 dim, float64 f); - - -/* Floor all non-0 elements of v[0..dim-1] to min value of f */ -void vector_nz_floor(vector_t v, int32 dim, float64 f); - - -/* - * Normalize the elements of the given vector so that they sum to 1.0. If the sum is 0.0 - * to begin with, the vector is left untouched. Return value: The normalization factor. - */ -float64 vector_sum_norm(vector_t v, int32 dim); - - -/* Print vector in one line, in %11.4e format, terminated by newline */ -void vector_print(FILE *fp, vector_t v, int32 dim); - - -/* Return TRUE iff given vector is all 0.0 */ -int32 vector_is_zero (float32 *vec, /* In: Vector to be checked */ - int32 len); /* In: Length of above vector */ - -#endif /* VECTOR_H */ diff --git a/src/featex.c b/src/featex.c index 984c63e..5225ada 100644 --- a/src/featex.c +++ b/src/featex.c @@ -1,30 +1,21 @@ - -// featex.c - PocketSphinx phonetic feature extraction for intelligibility prediction and remediation -// by James Salsman, July-August 2017 -// released under the MIT open source license - -// #define INFILENAME "featex.raw" #define FRATE 65 #define MODELDIR "/usr/local/share/pocketsphinx/model/en-us/en-us" -// #define DICTNAME "combo.dict" - -#include -#include "ps_alignment.h" - -#include "state_align_search.h" -#include "pocketsphinx_internal.h" -#include "ps_search.h" #include #include #include #include #include - #include #include #include +#include "ps_alignment.h" +#include +#include "state_align_search.h" +#include "pocketsphinx_internal.h" +#include "ps_search.h" + const char *argp_program_version = "Featex 0.1"; From 4380895b6de6eac90512dea472863f1af7c734dd Mon Sep 17 00:00:00 2001 From: Joshua Arulsamy Date: Tue, 26 May 2020 20:52:25 -0600 Subject: [PATCH 5/9] Fixed too long help section. --- src/featex.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/featex.c b/src/featex.c index 5225ada..65e9ae7 100644 --- a/src/featex.c +++ b/src/featex.c @@ -44,7 +44,7 @@ static struct argp_option options[] = {"phonemes", 'p', 0, 0, "Toggle play phonemes"}, {"triphones", 't', 0, 0, "Toggle play triphones"}, {"diphones", 'd', 0, 0, "Toggle play diphones"}, - {"phrase", 'P', "'QUOTE SURROUNDED PHRASE'", 0, "Input phrase"}, + {"phrase", 'P', "'PHRASE'", 0, "Input phrase"}, {0}}; /* From 7f44de15c72e181af3d5e851a9f445dd4a54b38e Mon Sep 17 00:00:00 2001 From: Joshua Arulsamy Date: Tue, 26 May 2020 20:58:52 -0600 Subject: [PATCH 6/9] Updated compiling and usage instructions. --- README.md | 143 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 98 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index a0489be..5269c98 100644 --- a/README.md +++ b/README.md @@ -1,48 +1,101 @@ # featex PocketSphinx phonetic feature extraction for intelligibility prediction and remediation -To compile and run, you need to install CMU PocketSphinx, e.g., on redhat/centos/fedora: - - sudo yum install svn autoconf libtool automake bison python-devel swig - -or, on debian/ubuntu/mint: - - sudo apt-get install subversion autoconf libtool automake bison python-dev swig - -then: - - cd - mkdir src - cd src - mkdir ps - cd ps - svn checkout svn://svn.code.sf.net/p/cmusphinx/code/trunk/sphinxbase - cd sphinxbase - ./autogen.sh - make - sudo make install - cd .. - svn checkout svn://svn.code.sf.net/p/cmusphinx/code/trunk/pocketsphinx - cd pocketsphinx - ./autogen.sh - make - sudo make install - - cd test/regression - ./test-lm.sh - -This should say, "All sub-tests passed" - - cd ~/src/ps - git clone https://github.com/jsalsman/featex - cd featex - - gcc -I /usr/local/include/pocketsphinx -I /usr/local/include/sphinxbase \ - -I ~/src/ps/pocketsphinx/src/libpocketsphinx \ - -o featex featex.c -lpocketsphinx -lsphinxbase -lm - - LD_LIBRARY_PATH=/usr/local/lib ./featex we drank tea in the afternoon and watched tv - -Make sure those `-I` include paths and the `#define MODELDIR` directive near the top of featex.c correctly identify where the include files, libraries, and en-us model files were actually installed. The numeric feature stream goes to standard output, and verbose debugging output goes to stderr, so in production you might likely run it with `2>/dev/null`. If you don't want to prepend the LD_LIBRARY_PATH, which is necessary on redhat but not debian variants, see e.g. https://serverfault.com/a/372998 - -The file `Spoken-English-Intelligibility-Remediation.pdf` -- also at http://arxiv.org/abs/1709.01713 -- has more information. +### Dependencies + +#### Packages + +To compile and run, you need to install CMU PocketSphinx +On Redhat/CentOS/Fedora: + + sudo yum install svn autoconf libtool automake bison python-devel swig gcc make + +or on Debian based systems: + + sudo apt-get install subversion autoconf libtool automake bison python-dev swig gcc make + +#### Sphinxbase and Pocketsphinx + +Install both sphinxbase and pocketsphinx. Go to any temp directory and execute the following. + +Sphinxbase: + + $ svn checkout svn://svn.code.sf.net/p/cmusphinx/code/trunk/sphinxbase + $ cd sphinxbase + $ ./autogen.sh + $ make + $ sudo make install + +Pocketsphinx: + + $ svn checkout svn://svn.code.sf.net/p/cmusphinx/code/trunk/ pocketsphinx + $ cd pocketsphinx + $ ./autogen.sh + $ make + $ sudo make install + +Run the builtin test: + + $ cd test/regression + $ ./test-lm.sh + +This should say, "All sub-tests passed" + +### Compiling + +Clone the repository: + + $ git clone https://github.com/jarulsamy/featex + +Create a build directory and `cd` into it: + + $ cd featex + $ mkdir build + $ cd build + +Generate the build files with `cmake`: + + $ cmake .. + +Finally, compile: + + $ make + +>The output binary, `featex.o` should be in the `bin/` directory. + + +### Usage + +View the built-in help with: + + $ ./featex --help + + Usage: featex.o [OPTION...] + featex -- PocketSphinx phonetic feature extraction for intelligibility + prediction and remediation + + -c, --combo=COMBO_PATH Path to combo.dict. + -d, --diphones Toggle play diphones + -i, --infile=INFILE_PATH Path to input raw file. + -p, --phonemes Toggle play phonemes + -P, --phrase='PHRASE' Input phrase + -t, --triphones Toggle play triphones + -u, --utterance Toggle play utterances + -w, --word Toggle play words + -?, --help Give this help list + --usage Give a short usage message + -V, --version Print program version + + Mandatory or optional arguments to long options are also mandatory or optional + for any corresponding short options. + + Report bugs to + +>The input phrase **MUST** be surrounded by quotes. + +>If you run into any missing libs while trying to run, try this: +>export LD_LIBRARY_PATH=/usr/local/lib + +The numeric feature stream goes to standard output, and verbose debugging output goes to stderr, so in production you will likely have to run it with `2>/dev/null`. If you don't want to prepend the LD_LIBRARY_PATH, which is necessary on redhat but not debian variants, see e.g. https://serverfault.com/a/372998 + +More info is documented in this [paper](assets/Spoken-English-Intelligibility-Remediation.pdf), and also [here](http://arxiv.org/abs/1709.01713). From ec384f6f5e627f960b2f7288795229a4e352ed4f Mon Sep 17 00:00:00 2001 From: Joshua Arulsamy Date: Tue, 26 May 2020 21:02:39 -0600 Subject: [PATCH 7/9] Removed extraneous headerlist variable --- src/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 586bc64..ce0130c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,5 +1,5 @@ # Add all the src cpp files with the headers -add_executable(featex.o featex.c ${HEADER_LIST}) +add_executable(featex.o featex.c) # Ensure everything in `include` is included. target_include_directories(featex.o PUBLIC ../include) @@ -9,4 +9,4 @@ target_include_directories(featex.o PUBLIC "/usr/local/include/sphinxbase") target_link_libraries(featex.o pocketsphinx sphinxbase m) # IDEs should put the headers in a nice place -source_group(TREE "${PROJECT_SOURCE_DIR}/include" PREFIX "Header Files" FILES ${HEADER_LIST}) +source_group(TREE "${PROJECT_SOURCE_DIR}/include" PREFIX "Header Files" FILES) From 6f45454cd543a548588434a578af179567bce78e Mon Sep 17 00:00:00 2001 From: Joshua Arulsamy Date: Tue, 26 May 2020 21:18:06 -0600 Subject: [PATCH 8/9] Added more usage info --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 5269c98..3f19433 100644 --- a/README.md +++ b/README.md @@ -99,3 +99,11 @@ View the built-in help with: The numeric feature stream goes to standard output, and verbose debugging output goes to stderr, so in production you will likely have to run it with `2>/dev/null`. If you don't want to prepend the LD_LIBRARY_PATH, which is necessary on redhat but not debian variants, see e.g. https://serverfault.com/a/372998 More info is documented in this [paper](assets/Spoken-English-Intelligibility-Remediation.pdf), and also [here](http://arxiv.org/abs/1709.01713). + +A demo can be run using `assets/combo.dict` and `assets/featex.raw`: + + $ ./featex.o -c assets/combo.dict -i assets/featex.raw -P 'we drank tea in the afternoon and watched tv` + +If you only want to see the end result and filter out all the debug data, redirect `stderr`: + + $ ./featex.o -c assets/combo.dict -i assets/featex.raw -P 'we drank tea in the afternoon and watched tv` 2> /dev/null From 500c554bcc9c2f0ac87cbc9f86fd1dfacbf20de0 Mon Sep 17 00:00:00 2001 From: Joshua Arulsamy Date: Thu, 28 May 2020 11:26:15 -0600 Subject: [PATCH 9/9] Fixed malformed URL --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3f19433..50ec4b5 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Sphinxbase: Pocketsphinx: - $ svn checkout svn://svn.code.sf.net/p/cmusphinx/code/trunk/ pocketsphinx + $ svn checkout svn://svn.code.sf.net/p/cmusphinx/code/trunk/pocketsphinx $ cd pocketsphinx $ ./autogen.sh $ make