Improve CW decoding

* Introduce the hysteresis option, in order the CW demodulator to adjust
properly the plateau length based on the WPM and any filtering that can
be used before

* Instead for a frame per word, now the CW decoder waits for 10 long
spaces before it commits a frame. With this way many words are placed on
the same frame telemetry decoding is easier

grc/satnogs_cw_to_symbol.xml

@@ -3,7 +3,7 @@
   <name>CW to Symbols</name>
   <key>satnogs_cw_to_symbol</key>
   <import>import satnogs</import>
-	<make>satnogs.cw_to_symbol($sampling_rate, $threshold, $conf_level, $wpm)</make>
+  <make>satnogs.cw_to_symbol($sampling_rate, $threshold, $conf_level, $wpm, $hysteresis)</make>
   <callback>set_act_threshold($threshold)</callback>
 
   <param>
@@ -33,6 +33,13 @@
     <type>int</type>
   </param>
   
+  <param>
+    <name>Hysteresis</name>
+    <key>hysteresis</key>
+    <value>0</value>
+    <type>int</type>
+  </param>
+
   <sink>
     <name>act_threshold</name>
     <type>message</type>

include/satnogs/cw_to_symbol.h

@@ -24,6 +24,9 @@
 #include <satnogs/api.h>
 #include <gnuradio/sync_block.h>
 
+#define MIN_WPM 5
+#define MAX_WPM 50
+
 namespace gr
 {
   namespace satnogs
@@ -65,10 +68,19 @@ namespace gr
        * symbols
        *
        * @param wpm Morse code Words per Minute
+       *
+       * @param hysteresis this value represents the hysteresis of a possible
+       * filter used before the CW to Symbol block. For example if there is
+       * a moving average filter with x taps, the full power of the signal
+       * will be available after x samples. The total length of samples with
+       * maximum power will be 2*x less. Because the block searches for plateaus
+       * with proper durations, this filtering hysteresis should be known.
+       * If no such a filter is used, the hysteresis value should be set to zero.
        */
       static cw_to_symbol::sptr
       make (double sampling_rate, float threshold, float conf_level = 0.9,
-            size_t wpm = 20);
+            size_t wpm = 20,
+            size_t hysteresis = 0);
 
       virtual void
       set_act_threshold (float thrld) = 0;

include/satnogs/morse.h

@@ -2,7 +2,7 @@
 /*
  * gr-satnogs: SatNOGS GNU Radio Out-Of-Tree Module
  *
- *  Copyright (C) 2016,2017
+ *  Copyright (C) 2016-2018
  *  Libre Space Foundation <http://librespacefoundation.org/>
  *
  *  This program is free software: you can redistribute it and/or modify
@@ -22,8 +22,6 @@
 #ifndef INCLUDE_SATNOGS_MORSE_H_
 #define INCLUDE_SATNOGS_MORSE_H_
 
-#define MIN_WPM 5
-#define MAX_WPM 30
 /**
  * The different Morse symbols
  */
@@ -32,7 +30,8 @@ typedef enum {
   MORSE_DASH,           //!< Morse dash (-) symbol
   MORSE_INTRA_SPACE,    //!< Space between dot and dash symbols
   MORSE_S_SPACE,        //!< Morse short space between characters
-  MORSE_L_SPACE         //!<Morse long space between words
+  MORSE_L_SPACE,        //!<Morse long space between words
+  MORSE_END_MSG_SPACE   //!< End of message
 } morse_symbol_t;
 
 

lib/cw_to_symbol_impl.cc

@@ -31,30 +31,32 @@
 #include <satnogs/utils.h>
 #include <boost/version.hpp>
 #if BOOST_VERSION >= 106500
-  #include <boost/integer/common_factor.hpp>
+#include <boost/integer/common_factor.hpp>
 #else
-  #include <boost/math/common_factor.hpp>
+#include <boost/math/common_factor.hpp>
 #endif
 #include <volk/volk.h>
 
 namespace gr
 {
-  namespace satnogs
+namespace satnogs
-  {
+{
 
-    cw_to_symbol::sptr
+cw_to_symbol::sptr
-    cw_to_symbol::make (double sampling_rate, float threshold, float conf_level,
+cw_to_symbol::make (double sampling_rate, float threshold, float conf_level,
-                        size_t wpm)
+                    size_t wpm, size_t hysteresis)
-    {
+{
   return gnuradio::get_initial_sptr (
-          new cw_to_symbol_impl (sampling_rate, threshold, conf_level, wpm));
+      new cw_to_symbol_impl (sampling_rate, threshold, conf_level, wpm,
-    }
+                             hysteresis));
+}
 
-    /*
+/*
  * The private constructor
  */
-    cw_to_symbol_impl::cw_to_symbol_impl (double sampling_rate, float threshold,
+cw_to_symbol_impl::cw_to_symbol_impl (double sampling_rate, float threshold,
-                                          float conf_level, size_t wpm) :
+                                      float conf_level, size_t wpm,
+                                      size_t hysteresis) :
         gr::sync_block ("cw_to_symbol",
                         gr::io_signature::make (1, 1, sizeof(float)),
                         gr::io_signature::make (0, 0, 0)),
@@ -62,15 +64,15 @@ namespace gr
         d_act_thrshld (threshold),
         d_confidence_level (conf_level),
         d_dot_samples ((1.2 / wpm) * sampling_rate),
-            d_window_size(0),
+        d_window_size (0),
-            d_window_cnt(0),
+        d_window_cnt (0),
-            d_dot_windows_num(0),
+        d_idle_cnt (0),
+        d_dot_windows_num (0),
         d_dec_state (NO_SYNC),
         d_prev_space_symbol (true)
-    {
+{
   if (wpm < MIN_WPM) {
-        throw std::invalid_argument (
+    throw std::invalid_argument ("Decoder can not handle such low WPM setting");
-            "Decoder can not handle such low WPM setting");
   }
 
   if (wpm > MAX_WPM) {
@@ -83,6 +85,10 @@ namespace gr
         "Confidence level should be in the range [0.5, 1.0]");
   }
 
+  if(hysteresis > d_dot_samples / 4) {
+    throw std::invalid_argument ("Too large hysteresis value");
+  }
+
   message_port_register_in (pmt::mp ("act_threshold"));
   message_port_register_out (pmt::mp ("out"));
 
@@ -92,6 +98,13 @@ namespace gr
       boost::bind (&cw_to_symbol_impl::set_act_threshold_msg_handler, this,
                    _1));
 
+  /*
+   * Reconsider the dot oulse duration based on the confidence level
+   * and hysteresis
+   */
+  d_dot_samples = d_dot_samples - 2 * (hysteresis * conf_level);
+
+
   /*
    * Try to split the CW pulses in smaller windows for detecting faster
    * a false alarm. As we use the window size for setting the history
@@ -99,18 +112,18 @@ namespace gr
    */
   size_t i = 1;
   d_window_size = d_dot_samples / i;
-      while(d_window_size > 64) {
+  while (d_window_size > 64) {
     i++;
     d_window_size = d_dot_samples / i;
   }
 
   /* NOTE: The dot duration should be a perfect multiple of the window */
-      while(d_dot_samples % d_window_size != 0) {
+  while (d_dot_samples % d_window_size != 0) {
     d_window_size++;
   }
 
-      LOG_WARN("Dot symbol samples: %lu", d_dot_samples);
+  LOG_DEBUG("Dot symbol samples: %lu", d_dot_samples);
-      LOG_WARN("Window size: %lu", d_window_size);
+  LOG_DEBUG("Window size: %lu", d_window_size);
 
   /* Set the duration of each symbol in multiples of the window size */
   d_dot_windows_num = d_dot_samples / d_window_size;
@@ -122,150 +135,157 @@ namespace gr
       / (d_window_size * sizeof(float));
   set_alignment (std::max (1, alignment_multiple));
 
-      d_const_val = (float *) volk_malloc(d_window_size * sizeof(float),
+  d_const_val = (float *) volk_malloc (d_window_size * sizeof(float),
                                        volk_get_alignment ());
-      d_tmp = (float *) volk_malloc(d_window_size * sizeof(float),
+  d_tmp = (float *) volk_malloc (d_window_size * sizeof(float),
                                  volk_get_alignment ());
   d_out = (int32_t *) volk_malloc (d_window_size * sizeof(int32_t),
                                    volk_get_alignment ());
 
-      if(!d_const_val || !d_tmp || !d_out) {
+  if (!d_const_val || !d_tmp || !d_out) {
-        throw std::runtime_error("cw_to_symbol: Could not allocate memory");
+    throw std::runtime_error ("cw_to_symbol: Could not allocate memory");
   }
 
-      for(i = 0; i < d_window_size; i++) {
+  for (i = 0; i < d_window_size; i++) {
     d_const_val[i] = threshold;
   }
-      set_history(d_window_size);
+  set_history (d_window_size);
-    }
+}
 
-    inline void
+inline void
-    cw_to_symbol_impl::send_symbol_msg (morse_symbol_t s)
+cw_to_symbol_impl::send_symbol_msg (morse_symbol_t s)
-    {
+{
-      if(s == MORSE_S_SPACE || s == MORSE_L_SPACE) {
+  if (s == MORSE_S_SPACE || s == MORSE_L_SPACE) {
     d_prev_space_symbol = true;
   }
-      else{
+  else {
     d_prev_space_symbol = false;
   }
   message_port_pub (pmt::mp ("out"), pmt::from_long (s));
-    }
+}
 
-    inline bool
+inline bool
-    cw_to_symbol_impl::check_conf_level(size_t cnt, size_t target)
+cw_to_symbol_impl::check_conf_level (size_t cnt, size_t target)
-    {
+{
-      return ((float)cnt > target * d_confidence_level);
+  return ((float) cnt > target * d_confidence_level);
-    }
+}
 
-    /*
+/*
  * Our virtual destructor.
  */
-    cw_to_symbol_impl::~cw_to_symbol_impl ()
+cw_to_symbol_impl::~cw_to_symbol_impl ()
-    {
+{
   volk_free (d_const_val);
   volk_free (d_tmp);
   volk_free (d_out);
-    }
+}
 
-    inline void
+inline void
-    cw_to_symbol_impl::set_idle ()
+cw_to_symbol_impl::set_idle ()
-    {
+{
   d_dec_state = NO_SYNC;
   d_window_cnt = 0;
-    }
+  d_idle_cnt = 0;
+}
 
-    inline void
+inline void
-    cw_to_symbol_impl::set_short_on ()
+cw_to_symbol_impl::set_short_on ()
-    {
+{
   d_dec_state = SEARCH_DOT;
   d_window_cnt = 1;
-    }
+}
 
-    inline void
+inline void
-    cw_to_symbol_impl::set_long_on ()
+cw_to_symbol_impl::set_long_on ()
-    {
+{
   d_dec_state = SEARCH_DASH;
-    }
+}
 
-    inline void
+inline void
-    cw_to_symbol_impl::set_search_space ()
+cw_to_symbol_impl::set_search_space ()
-    {
+{
   d_dec_state = SEARCH_SPACE;
   d_window_cnt = 1;
-    }
+}
 
-    void
+void
-    cw_to_symbol_impl::set_act_threshold_msg_handler (pmt::pmt_t msg)
+cw_to_symbol_impl::set_act_threshold_msg_handler (pmt::pmt_t msg)
-    {
+{
   if (pmt::is_pair (msg)) {
     set_act_threshold (pmt::to_double (pmt::cdr (msg)));
   }
-    }
+}
 
-    int
+int
-    cw_to_symbol_impl::work (int noutput_items,
+cw_to_symbol_impl::work (int noutput_items,
                          gr_vector_const_void_star &input_items,
                          gr_vector_void_star &output_items)
-    {
+{
   bool triggered;
   size_t i;
   const float *in_old = (const float *) input_items[0];
-      const float *in = in_old + history() - 1;
+  const float *in = in_old + history () - 1;
 
-      if(noutput_items < 0) {
+  if (noutput_items < 0) {
     return noutput_items;
   }
 
   /* During idle state search for a possible trigger */
-      if(d_dec_state == NO_SYNC) {
+  if (d_dec_state == NO_SYNC) {
-        for(i = 0; i < (size_t)noutput_items; i++) {
+    for (i = 0; i < (size_t) noutput_items; i++) {
       /*
        * Clamp the input so the window mean is not affected by strong spikes
        * Good luck understanding this black magic shit!
        */
-          triggered = is_triggered(in_old + i, d_window_size);
+      triggered = is_triggered (in_old + i, d_window_size);
-          if(triggered) {
+      if (triggered) {
         LOG_DEBUG("Triggered!");
-            set_short_on();
+        set_short_on ();
-            return i+1;
+        return i + 1;
       }
     }
+    d_idle_cnt += noutput_items / d_window_size;
+    if(d_idle_cnt > 10 * d_long_pause_windows_num) {
+      send_symbol_msg (MORSE_END_MSG_SPACE);
+      d_idle_cnt = 0;
+    }
     return noutput_items;
   }
 
   /* From now one, we handle the input in multiples of a window */
-      for (i = 0; i < (size_t)noutput_items / d_window_size; i++) {
+  for (i = 0; i < (size_t) noutput_items / d_window_size; i++) {
-        triggered = is_triggered(in + i * d_window_size, d_window_size);
+    triggered = is_triggered (in + i * d_window_size, d_window_size);
-        switch(d_dec_state) {
+    switch (d_dec_state)
+      {
       case SEARCH_DOT:
-            if(triggered) {
+        if (triggered) {
           d_window_cnt++;
-              if(d_window_cnt > d_dot_windows_num) {
+          if (d_window_cnt > d_dot_windows_num) {
-                set_long_on();
+            set_long_on ();
             LOG_DEBUG("Going to search for long sequence");
           }
         }
         else {
-              if(check_conf_level(d_window_cnt, d_dot_windows_num)) {
+          if (check_conf_level (d_window_cnt, d_dot_windows_num)) {
             LOG_DEBUG("DOT");
-                send_symbol_msg(MORSE_DOT);
+            send_symbol_msg (MORSE_DOT);
           }
           LOG_DEBUG("Going to search for space: win cnt %lu", d_window_cnt);
           set_search_space ();
         }
         break;
       case SEARCH_DASH:
-            if(triggered) {
+        if (triggered) {
           d_window_cnt++;
         }
-            else{
+        else {
-              if(check_conf_level(d_window_cnt, d_dash_windows_num)) {
+          if (check_conf_level (d_window_cnt, d_dash_windows_num)) {
             LOG_DEBUG("DASH");
-                send_symbol_msg(MORSE_DASH);
+            send_symbol_msg (MORSE_DASH);
           }
-              else{
+          else {
             LOG_DEBUG("DOT");
-                send_symbol_msg(MORSE_DOT);
+            send_symbol_msg (MORSE_DOT);
           }
           set_search_space ();
           LOG_DEBUG("Going to search for space");
@@ -273,23 +293,23 @@ namespace gr
         break;
       case SEARCH_SPACE:
         if (triggered) {
-              if(check_conf_level(d_window_cnt, d_long_pause_windows_num)) {
+          if (check_conf_level (d_window_cnt, d_long_pause_windows_num)) {
             LOG_DEBUG("LONG SPACE");
-                send_symbol_msg(MORSE_L_SPACE);
+            send_symbol_msg (MORSE_L_SPACE);
           }
-              else if(check_conf_level(d_window_cnt, d_short_pause_windows_num)){
+          else if (check_conf_level (d_window_cnt, d_short_pause_windows_num)) {
             LOG_DEBUG("SHORT SPACE");
-                send_symbol_msg(MORSE_S_SPACE);
+            send_symbol_msg (MORSE_S_SPACE);
           }
-              set_short_on();
+          set_short_on ();
           LOG_DEBUG("Going to search for dot");
         }
-            else{
+        else {
           d_window_cnt++;
-              if(d_window_cnt > d_long_pause_windows_num) {
+          if (d_window_cnt > d_long_pause_windows_num) {
             LOG_DEBUG("LONG SPACE");
-                send_symbol_msg(MORSE_L_SPACE);
+            send_symbol_msg (MORSE_L_SPACE);
-                set_idle();
+            set_idle ();
             LOG_DEBUG("Going to idle");
             return (i + 1) * d_window_size;
           }
@@ -300,19 +320,19 @@ namespace gr
       }
   }
   return i * d_window_size;
-    }
+}
 
-    /**
+/**
  * Sets a new activation threshold.
  * @param thrhld the new threshold.
  */
-    void
+void
-    cw_to_symbol_impl::set_act_threshold (float thrhld)
+cw_to_symbol_impl::set_act_threshold (float thrhld)
-    {
+{
   d_act_thrshld = thrhld;
-    }
+}
 
-    /**
+/**
  * Clamps the input and performs at the same time binary slicing.
  * With this way, a decision based on moving average is not affected
  * by strong peaks.
@@ -320,33 +340,33 @@ namespace gr
  * @param in the input signal
  * @param len number of samples to process
  */
-    inline void
+inline void
-    cw_to_symbol_impl::clamp_input (int32_t* out, const float* in, size_t len)
+cw_to_symbol_impl::clamp_input (int32_t* out, const float* in, size_t len)
-    {
+{
-      volk_32f_x2_subtract_32f(d_tmp, in, d_const_val, len);
+  volk_32f_x2_subtract_32f (d_tmp, in, d_const_val, len);
-      volk_32f_binary_slicer_32i(d_out, d_tmp, len);
+  volk_32f_binary_slicer_32i (d_out, d_tmp, len);
-    }
+}
 
-    static inline int32_t
+static inline int32_t
-    hadd (const int32_t* in, size_t len)
+hadd (const int32_t* in, size_t len)
-    {
+{
   size_t i;
   int32_t cnt = 0;
-      for(i = 0; i < len; i++) {
+  for (i = 0; i < len; i++) {
     cnt += in[i];
   }
   return cnt;
-    }
+}
 
-    inline bool
+inline bool
-    cw_to_symbol_impl::is_triggered (const float* in, size_t len)
+cw_to_symbol_impl::is_triggered (const float* in, size_t len)
-    {
+{
   int32_t cnt;
-      clamp_input(d_out, in, len);
+  clamp_input (d_out, in, len);
-      cnt = hadd(d_out, len);
+  cnt = hadd (d_out, len);
-      return (cnt >= (int32_t)(d_window_size * d_confidence_level)) ? true : false;
+  return (cnt >= (int32_t) (d_window_size * d_confidence_level)) ? true : false;
-    }
+}
 
-  } /* namespace satnogs */
+} /* namespace satnogs */
 } /* namespace gr */
 

lib/cw_to_symbol_impl.h

@@ -45,6 +45,7 @@ namespace gr
       size_t d_dot_samples;
       size_t d_window_size;
       size_t d_window_cnt;
+      size_t d_idle_cnt;
       size_t d_dot_windows_num;
       size_t d_dash_windows_num;
       size_t d_short_pause_windows_num;
@@ -84,7 +85,7 @@ namespace gr
 
     public:
       cw_to_symbol_impl (double sampling_rate, float threshold,
-                         float conf_level, size_t wpm);
+                         float conf_level, size_t wpm, size_t hysteresis);
       ~cw_to_symbol_impl ();
 
       // Where all the action really happens

lib/morse_decoder_impl.cc

@@ -28,21 +28,20 @@
 #include <satnogs/log.h>
 namespace gr
 {
-  namespace satnogs
+namespace satnogs
-  {
+{
 
-    morse_decoder::sptr
+morse_decoder::sptr
-    morse_decoder::make (char unrecognized_char, size_t min_frame_len)
+morse_decoder::make (char unrecognized_char, size_t min_frame_len)
-    {
+{
   return gnuradio::get_initial_sptr (
       new morse_decoder_impl (unrecognized_char, min_frame_len));
-    }
+}
 
-    void
+void
-    morse_decoder_impl::symbol_msg_handler (pmt::pmt_t msg)
+morse_decoder_impl::symbol_msg_handler (pmt::pmt_t msg)
-    {
+{
   bool res = false;
-      std::string str;
   morse_symbol_t s;
   s = (morse_symbol_t) pmt::to_long (msg);
 
@@ -68,16 +67,21 @@ namespace gr
         res = true;
         break;
       }
-          str = d_morse_tree.get_word ();
+      d_str = d_str.append(d_morse_tree.get_word ());
-          if (str.length () > d_min_frame_len) {
+      d_str = d_str.append(" ");
       d_morse_tree.reset ();
-            message_port_pub (pmt::mp ("out"),
-                              pmt::make_blob (str.c_str (), str.length ()));
-          }
       break;
     case MORSE_INTRA_SPACE:
       /*Ignore it */
       break;
+    case MORSE_END_MSG_SPACE:
+      if (d_str.length () > d_min_frame_len) {
+        d_morse_tree.reset ();
+        message_port_pub (pmt::mp ("out"),
+                          pmt::make_blob (d_str.c_str (), d_str.length ()));
+        d_str = "";
+      }
+      break;
     default:
       LOG_ERROR("Unknown Morse symbol");
       return;
@@ -89,33 +93,34 @@ namespace gr
    */
   if (!res) {
     if (d_morse_tree.get_max_word_len () == d_morse_tree.get_word_len ()) {
-          str = d_morse_tree.get_word ();
+      d_str = d_morse_tree.get_word ();
       d_morse_tree.reset ();
       message_port_pub (pmt::mp ("out"),
-                            pmt::make_blob (str.c_str (), str.length ()));
+                        pmt::make_blob (d_str.c_str (), d_str.length ()));
-        }
     }
   }
+}
 
-    /*
+/*
  * The private constructor
  */
-    morse_decoder_impl::morse_decoder_impl (char unrecognized_char,
+morse_decoder_impl::morse_decoder_impl (char unrecognized_char,
                                         size_t min_frame_len) :
         gr::block ("morse_decoder", gr::io_signature::make (0, 0, 0),
                    gr::io_signature::make (0, 0, 0)),
         d_morse_tree (unrecognized_char),
-            d_min_frame_len (min_frame_len)
+        d_min_frame_len (min_frame_len),
+        d_str("")
 
-    {
+{
   /* Register the input and output msg handler */
   message_port_register_in (pmt::mp ("in"));
   message_port_register_out (pmt::mp ("out"));
   set_msg_handler (
       pmt::mp ("in"),
       boost::bind (&morse_decoder_impl::symbol_msg_handler, this, _1));
-    }
+}
 
-  } /* namespace satnogs */
+} /* namespace satnogs */
 } /* namespace gr */
 

lib/morse_decoder_impl.h

@@ -24,24 +24,27 @@
 #include <satnogs/morse_decoder.h>
 #include <satnogs/morse_tree.h>
 
-namespace gr {
+namespace gr
-  namespace satnogs {
+{
+namespace satnogs
+{
 
-    class morse_decoder_impl : public morse_decoder
+class morse_decoder_impl : public morse_decoder
-    {
+{
-     private:
+
+public:
+  morse_decoder_impl (char unrecognized_char, size_t min_frame_len);
+
+private:
   morse_tree            d_morse_tree;
   size_t                d_min_frame_len;
+  std::string           d_str;
 
   void
-      symbol_msg_handler(pmt::pmt_t msg);
+  symbol_msg_handler (pmt::pmt_t msg);
+};
 
-     public:
+} // namespace satnogs
-      morse_decoder_impl(char unrecognized_char, size_t min_frame_len);
-
-    };
-
-  } // namespace satnogs
 } // namespace gr
 
 #endif /* INCLUDED_SATNOGS_MORSE_DECODER_IMPL_H */