]> git.ipfire.org Git - thirdparty/freeswitch.git/commitdiff
FS-11453 [mod_rayo] add support for FS JSON ASR result.
authorChris Rienzo <chris@signalwire.com>
Thu, 11 Oct 2018 18:45:11 +0000 (18:45 +0000)
committerChris Rienzo <chris@signalwire.com>
Thu, 20 Dec 2018 15:17:57 +0000 (10:17 -0500)
src/mod/event_handlers/mod_rayo/nlsml.c
src/mod/event_handlers/mod_rayo/nlsml.h
src/mod/event_handlers/mod_rayo/rayo_input_component.c
src/mod/event_handlers/mod_rayo/test_nlsml/Makefile

index 79f8157e9fb74ed186083aba236b5e6b957192fa..752ddc3efa44a8409d58cc4cb7d7b6cb46ed9892 100644 (file)
@@ -1,6 +1,6 @@
 /*
  * mod_rayo for FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application
- * Copyright (C) 2013-2014, Grasshopper
+ * Copyright (C) 2013-2018, Grasshopper
  *
  * Version: MPL 1.1
  *
@@ -30,6 +30,7 @@
 #include <iksemel.h>
 
 #include "nlsml.h"
+#include "iks_helpers.h"
 
 struct nlsml_parser;
 
@@ -417,51 +418,63 @@ static int isdtmf(const char digit)
 }
 
 /**
- * Construct an NLSML result for digit match
- * @param digits the matching digits
+ * Construct an NLSML result for match
+ * @param match the matching digits or text
  * @param interpretation the optional digit interpretation
+ * @param mode dtmf or speech
+ * @param confidence 0-100
  * @return the NLSML <result>
  */
-iks *nlsml_create_dtmf_match(const char *digits, const char *interpretation)
+iks *nlsml_create_match(const char *match, const char *interpretation, const char *mode, int confidence)
 {
        iks *result = iks_new("result");
        iks_insert_attrib(result, "xmlns", NLSML_NS);
        iks_insert_attrib(result, "xmlns:xf", "http://www.w3.org/2000/xforms");
-       if (!zstr(digits)) {
-               int first = 1;
-               int i;
-               int num_digits = strlen(digits);
-               switch_stream_handle_t stream = { 0 };
-
+       if (!zstr(match)) {
                iks *interpretation_node = iks_insert(result, "interpretation");
                iks *input_node = iks_insert(interpretation_node, "input");
                iks *instance_node = iks_insert(interpretation_node, "instance");
-               iks_insert_attrib(input_node, "mode", "dtmf");
-               iks_insert_attrib(input_node, "confidence", "100");
-
-               SWITCH_STANDARD_STREAM(stream);
-               for (i = 0; i < num_digits; i++) {
-                       if (isdtmf(digits[i])) {
-                               if (first) {
-                                       stream.write_function(&stream, "%c", digits[i]);
-                                       first = 0;
-                               } else {
-                                       stream.write_function(&stream, " %c", digits[i]);
-                               }
-                       }
-               }
-               iks_insert_cdata(input_node, stream.data, strlen(stream.data));
-
+               iks_insert_attrib(input_node, "mode", mode);
+               iks_insert_attrib_printf(input_node, "confidence", "%d", confidence);
+               iks_insert_cdata(input_node, match, strlen(match));
                if (zstr(interpretation)) {
-                       iks_insert_cdata(instance_node, stream.data, strlen(stream.data));
+                       iks_insert_cdata(instance_node, match, strlen(match));
                } else {
                        iks_insert_cdata(instance_node, interpretation, strlen(interpretation));
                }
-               switch_safe_free(stream.data);
        }
        return result;
 }
 
+/**
+ * Construct an NLSML result for match
+ * @param match the matching digits or text
+ * @param interpretation the optional digit interpretation
+ * @return the NLSML <result>
+ */
+iks *nlsml_create_dtmf_match(const char *digits, const char *interpretation)
+{
+       iks *result = NULL;
+       int first = 1;
+       int i;
+       int num_digits = strlen(digits);
+       switch_stream_handle_t stream = { 0 };
+       SWITCH_STANDARD_STREAM(stream);
+       for (i = 0; i < num_digits; i++) {
+               if (isdtmf(digits[i])) {
+                       if (first) {
+                               stream.write_function(&stream, "%c", digits[i]);
+                               first = 0;
+                       } else {
+                               stream.write_function(&stream, " %c", digits[i]);
+                       }
+               }
+       }
+       result = nlsml_create_match((const char *)stream.data, interpretation, "dtmf", 100);
+       switch_safe_free(stream.data);
+       return result;
+}
+
 /**
  * Initialize NLSML parser.  This function is not thread safe.
  */
index f5be079ba56c85f055be41d4c0c7b9c60b0c3a7b..475abd091e7a33930f9965d1a2962f9ff0d03cee 100644 (file)
@@ -44,6 +44,7 @@ extern void nlsml_destroy(void);
 enum nlsml_match_type nlsml_parse(const char *result, const char *uuid);
 iks *nlsml_normalize(const char *result);
 extern iks *nlsml_create_dtmf_match(const char *digits, const char *interpretation);
+extern iks *nlsml_create_match(const char *digits, const char *interpretation, const char *mode, int confidence);
 
 #endif
 
index dc7a71c9d9e85b5bb3ace0dfb890d4fdbe27c219..13311d544428fa7cdfb3b0a23b986deac193b41e 100644 (file)
@@ -876,6 +876,30 @@ static iks *start_timers_call_input_component(struct rayo_actor *component, stru
        return iks_new_iq_result(iq);
 }
 
+/**
+ * Get text / error from result
+ */
+static const char *get_detected_speech_result_text(cJSON *result_json, double *confidence, const char **error_text)
+{
+       const char *result_text = NULL;
+       const char *text = cJSON_GetObjectCstr(result_json, "text");
+       if (confidence) {
+               *confidence = 0.0;
+       }
+       if (!zstr(text)) {
+               cJSON *json_confidence = cJSON_GetObjectItem(result_json, "confidence");
+               if (json_confidence && json_confidence->valuedouble > 0.0) {
+                       *confidence = json_confidence->valuedouble;
+               } else {
+                       *confidence = 100.0;
+               }
+               result_text = text;
+       } else if (error_text) {
+               *error_text = cJSON_GetObjectCstr(result_json, "error");
+       }
+       return result_text;
+}
+
 /**
  * Handle speech detection event
  */
@@ -905,7 +929,50 @@ static void on_detected_speech_event(switch_event_t *event)
                        if (zstr(result)) {
                                rayo_component_send_complete(component, INPUT_NOMATCH);
                        } else {
-                               if (strchr(result, '<')) {
+                               if (result[0] == '{') {
+                                       // internal FS JSON format
+                                       cJSON *json_result = cJSON_Parse(result);
+                                       if (json_result) {
+                                               // examine result to determine what happened
+                                               double confidence = 0.0;
+                                               const char *error_text = NULL;
+                                               const char *result_text = NULL;
+                                               result_text = get_detected_speech_result_text(json_result, &confidence, &error_text);
+                                               if (!zstr(result_text)) {
+                                                       // got result... send as NLSML
+                                                       iks *result = nlsml_create_match(result_text, NULL, "speech", (int)confidence);
+                                                       /* notify of match */
+                                                       switch_log_printf(SWITCH_CHANNEL_UUID_LOG(uuid), SWITCH_LOG_DEBUG, "MATCH = %s\n", result_text);
+                                                       send_match_event(RAYO_COMPONENT(component), result);
+                                                       iks_delete(result);
+                                               } else if (zstr(error_text)) {
+                                                       // unknown error
+                                                       switch_log_printf(SWITCH_CHANNEL_UUID_LOG(uuid), SWITCH_LOG_WARNING, "No matching text nor error in result: %s!\n", result);
+                                                       rayo_component_send_complete(component, INPUT_NOMATCH);
+                                               } else if (!strcmp(error_text, "no_input")) {
+                                                       // no input error
+                                                       rayo_component_send_complete(component, INPUT_NOINPUT);
+                                               } else if (!strcmp(error_text, "no_match")) {
+                                                       // no match error
+                                                       rayo_component_send_complete(component, INPUT_NOMATCH);
+                                               } else {
+                                                       // generic error
+                                                       iks *response = rayo_component_create_complete_event(component, COMPONENT_COMPLETE_ERROR);
+                                                       iks *error = NULL;
+                                                       if ((error = iks_find(response, "complete"))) {
+                                                               if ((error = iks_find(error, "error"))) {
+                                                                       iks_insert_cdata(error, error_text, strlen(error_text));
+                                                               }
+                                                       }
+                                                       rayo_component_send_complete_event(component, response);
+                                               }
+                                               cJSON_Delete(json_result);
+                                       } else {
+                                               // failed to parse JSON result
+                                               switch_log_printf(SWITCH_CHANNEL_UUID_LOG(uuid), SWITCH_LOG_WARNING, "Failed to parse JSON result: %s!\n", result);
+                                               rayo_component_send_complete(component, INPUT_NOMATCH);
+                                       }
+                               } else if (strchr(result, '<')) {
                                        /* got an XML result */
                                        enum nlsml_match_type match_type = nlsml_parse(result, uuid);
                                        switch (match_type) {
index f363f55590705c9d6359c5b853eeeaa6fff902a7..8104690733a478a3e605521c92900fa44253271a 100644 (file)
@@ -3,7 +3,7 @@ BASE=../../../../..
 IKS_DIR=$(BASE)/libs/iksemel
 IKS_LA=$(IKS_DIR)/src/libiksemel.la
 LOCAL_CFLAGS += -I../ -I$(BASE)/libs/iksemel/include
-LOCAL_OBJS= $(PCRE_LA) $(IKS_LA) main.o ../nlsml.o
+LOCAL_OBJS= $(PCRE_LA) $(IKS_LA) main.o ../nlsml.o ../iks_helpers.o
 LOCAL_SOURCES= main.c
 include $(BASE)/build/modmake.rules
 
@@ -12,7 +12,7 @@ $(IKS_LA): $(IKS_DIR) $(IKS_DIR)/.update
        @$(TOUCH_TARGET)
 
 local_all:
-       libtool --mode=link gcc main.o ../nlsml.o -o test test_nlsml.la
-
+       libtool --mode=link gcc main.o ../nlsml.o ../iks_helpers.o  test_nlsml.la ../../../../../.libs/libfreeswitch.la ../../../../../libs/iksemel/src/.libs/libiksemel.a -lpcre -lssl -lcrypto -g -ggdb -O2 -pthread -o test
 local_clean:
        -rm test