ipaacalib/python/src/ipaaca/util/__init__.py

@@ -4,7 +4,7 @@
 #  "Incremental Processing Architecture
 #   for Artificial Conversational Agents".
 #
-# Copyright (c) 2009-2014 Social Cognitive Systems Group
+# Copyright (c) 2009-2022 Social Cognitive Systems Group
 #                         CITEC, Bielefeld University
 #
 # http://opensource.cit-ec.de/projects/ipaaca/
@@ -32,4 +32,4 @@
 
 from __future__ import division, print_function
 
-from notifier import ComponentNotifier
+from .notifier import ComponentNotifier

ipaacalib/python/src/ipaaca/util/logger.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# This file is part of IPAACA, the
+#  "Incremental Processing Architecture
+#   for Artificial Conversational Agents".
+#
+# Copyright (c) 2009-2022 Social Cognitive Systems Group
+#                         CITEC, Bielefeld University
+#
+# http://opensource.cit-ec.de/projects/ipaaca/
+# http://purl.org/net/ipaaca
+#
+# This file may be licensed under the terms of of the
+# GNU Lesser General Public License Version 3 (the ``LGPL''),
+# or (at your option) any later version.
+#
+# Software distributed under the License is distributed
+# on an ``AS IS'' basis, WITHOUT WARRANTY OF ANY KIND, either
+# express or implied. See the LGPL for the specific language
+# governing rights and limitations.
+#
+# You should have received a copy of the LGPL along with this
+# program. If not, go to http://www.gnu.org/licenses/lgpl.html
+# or write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# The development of this software was supported by the
+# Excellence Cluster EXC 277 Cognitive Interaction Technology.
+# The Excellence Cluster EXC 277 is a grant of the Deutsche
+# Forschungsgemeinschaft (DFG) in the context of the German
+# Excellence Initiative.
+
+from __future__ import division, print_function
+
+import datetime
+import subprocess
+import sys
+import threading
+import time
+import traceback
+import uuid
+
+import ipaaca
+import ipaaca.misc
+import six
+
+
+__all__ = [
+	'logger_send_ipaaca_logs',
+	'logger_set_log_filename',
+	'logger_set_module_name',
+	'logger_set_log_level',
+	'LOG_DEBUG',
+	'LOG_INFO',
+	'LOG_WARN',
+	'LOG_WARNING',
+	'LOG_ERROR',
+]
+
+
+LogLevel = ipaaca.misc.enum(
+	DEBUG = 0,
+	INFO = 1,
+	WARN = 2,
+	ERROR = 3,
+	SILENT = 4,
+)
+LOG_LEVEL_FROM_STRING_DICT = {
+	'DEBUG': LogLevel.DEBUG,
+	'INFO': LogLevel.INFO,
+	'WARN': LogLevel.WARN,
+	'WARNING': LogLevel.WARN,
+	'ERROR': LogLevel.ERROR,
+	'NONE': LogLevel.SILENT,
+	'SILENT': LogLevel.SILENT,
+}
+CURRENT_LOG_LEVEL = LogLevel.DEBUG
+
+LOGGER_LOCK = threading.RLock()
+
+MODULE_NAME = sys.argv[0]
+
+SEND_IPAACA_LOGS = True
+
+OUTPUT_BUFFER = None
+
+STANDARD_LOG_FILE_EXTENSION = '.log'
+
+LOG_MODES = ['append', 'timestamp', 'overwrite']
+
+
+def logger_set_log_filename(filename, existing=None):
+	global OUTPUT_BUFFER
+	if existing is not None and not existing in LOG_MODES:
+		raise Exception('Invalid log mode {mode} given. '
+			'Valid options are {options}.'.format(
+				mode=existing,
+				options=', '.join(LOG_MODES)))
+	with LOGGER_LOCK:
+		if OUTPUT_BUFFER is None:
+			OUTPUT_BUFFER = ipaaca.OutputBuffer('LogSender')
+		msg = ipaaca.Message('logcontrol')
+		msg.payload = {
+			'cmd': 'open_log_file',
+			'filename': filename}
+		if existing is not None:
+			msg.payload['existing'] = existing
+		OUTPUT_BUFFER.add(msg)
+
+
+def logger_set_module_name(name):
+	global MODULE_NAME
+	with LOGGER_LOCK:
+		MODULE_NAME = name
+
+
+def logger_send_ipaaca_logs(flag=True):
+	global SEND_IPAACA_LOGS
+	with LOGGER_LOCK:
+		SEND_IPAACA_LOGS = flag
+
+def logger_set_log_level(level=LogLevel.DEBUG):
+	global CURRENT_LOG_LEVEL
+	with LOGGER_LOCK:
+		if level in LogLevel._values:
+			CURRENT_LOG_LEVEL = level
+		elif isinstance(level, six.string_types) and level.upper() in LOG_LEVEL_FROM_STRING_DICT:
+			CURRENT_LOG_LEVEL = LOG_LEVEL_FROM_STRING_DICT[level.upper()]
+		else:
+			pass # leave previous setting untouched
+
+def LOG_IPAACA(lvl, text, now=0.0, fn='???', thread='???'):
+	global OUTPUT_BUFFER
+	uid = str(uuid.uuid4())[0:8]
+	with LOGGER_LOCK:
+		if OUTPUT_BUFFER is None:
+			OUTPUT_BUFFER = ipaaca.OutputBuffer('LogSender')
+		msg = ipaaca.Message('log')
+		msg.payload = {
+				'module': MODULE_NAME,
+				'function': fn,
+				'level': lvl,
+				'time':' %.3f'%now,
+				'thread': thread,
+				'uuid': uid,
+				'text': text,}
+		try:
+			OUTPUT_BUFFER.add(msg)
+		except Exception as e:
+			LOG_ERROR('Caught an exception while logging via ipaaca. '
+				+ ' str(e); ' 
+				+ traceback.format_exc())
+
+
+def LOG_CONSOLE(lvlstr, msg, fn_markup='', msg_markup='', now=0.0, fn='???', thread='???'):
+	if isinstance(msg, six.string_types):
+		lines = msg.split('\n')
+	else:
+		lines = [msg]
+	for line in lines:
+		text = lvlstr+' '+thread+' '+fn_markup+fn+'\033[m'+' '+msg_markup+str(line)+'\033[m'
+		print(text)
+		fn = ' '*len(fn)
+
+
+def LOG_ERROR(msg, now=None):
+	if CURRENT_LOG_LEVEL > LogLevel.ERROR: return
+	now = time.time() if now is None else now
+	f = sys._getframe(1)
+	classprefix = (f.f_locals['self'].__class__.__name__+'.') if 'self' in f.f_locals else ''
+	fn = classprefix + f.f_code.co_name
+	thread = threading.current_thread().getName()
+	with LOGGER_LOCK:
+		if SEND_IPAACA_LOGS: LOG_IPAACA('ERROR', msg, now=now, fn=fn, thread=thread)
+		LOG_CONSOLE('\033[38;5;9;1;4m[ERROR]\033[m', msg, fn_markup='\033[38;5;203m', msg_markup='\033[38;5;9;1;4m', now=now, fn=fn, thread=thread)
+
+
+def LOG_WARN(msg, now=None):
+	if CURRENT_LOG_LEVEL > LogLevel.WARN: return
+	now = time.time() if now is None else now
+	f = sys._getframe(1)
+	classprefix = (f.f_locals['self'].__class__.__name__+'.') if 'self' in f.f_locals else ''
+	fn = classprefix + f.f_code.co_name
+	thread = threading.current_thread().getName()
+	with LOGGER_LOCK:
+		if SEND_IPAACA_LOGS: LOG_IPAACA('WARN', msg, now=now, fn=fn, thread=thread)
+		LOG_CONSOLE('\033[38;5;208;1m[WARN]\033[m ', msg, fn_markup='\033[38;5;214m', msg_markup='\033[38;5;208;1m', now=now, fn=fn, thread=thread)
+
+
+LOG_WARNING = LOG_WARN
+
+
+def LOG_INFO(msg, now=None):
+	if CURRENT_LOG_LEVEL > LogLevel.INFO: return
+	now = time.time() if now is None else now
+	f = sys._getframe(1)
+	classprefix = (f.f_locals['self'].__class__.__name__+'.') if 'self' in f.f_locals else ''
+	fn = classprefix + f.f_code.co_name
+	thread = threading.current_thread().getName()
+	with LOGGER_LOCK:
+		if SEND_IPAACA_LOGS: LOG_IPAACA('INFO', msg, now=now, fn=fn, thread=thread)
+		LOG_CONSOLE('[INFO] ', msg, now=now, fn=fn, thread=thread)
+
+
+def LOG_DEBUG(msg, now=None):
+	if CURRENT_LOG_LEVEL > LogLevel.DEBUG: return
+	now = time.time() if now is None else now
+	f = sys._getframe(1)
+	classprefix = (f.f_locals['self'].__class__.__name__+'.') if 'self' in f.f_locals else ''
+	fn = classprefix + f.f_code.co_name
+	thread = threading.current_thread().getName()
+	with LOGGER_LOCK:
+		if SEND_IPAACA_LOGS: LOG_IPAACA('DEBUG', msg, now=now, fn=fn, thread=thread)
+		LOG_CONSOLE('\033[2m[DEBUG]\033[m', msg, fn_markup='\033[38;5;144m', msg_markup='\033[38;5;248m', now=now, fn=fn, thread=thread)
+
+
+class LoggerComponent(object):
+
+	def __init__(self, filename, log_mode='append'):
+		self.ib = ipaaca.InputBuffer('Logger', ['log', 'logcontrol'])
+		self.ib.register_handler(self._logger_handle_iu_event)
+		self.logfile = None
+		self.log_mode = log_mode
+		self.open_logfile(filename)
+		if self.logfile is None:
+			print('Logging to console only ...')
+		print('Press Ctrl-C at any time to terminate the logger.')
+
+	def open_logfile(self, filename):
+		with LOGGER_LOCK:
+			if filename is None or filename.strip() == '':
+				print('No log file name specified, not opening one.')
+				self.close_logfile()
+			else:
+				new_logfile = None
+				try:
+					# create dir first
+					ret = subprocess.call(
+							'mkdir -p `dirname ' + filename + '`',
+							shell=True)
+					# proceed with dir+file
+					if self.log_mode == 'timestamp':
+						t = datetime.datetime.now().strftime('%Y-%m-%d-%H%M%S')
+						if filename.endswith(STANDARD_LOG_FILE_EXTENSION):
+							# insert timestamp between filename and extension
+							# (only for standard extension)
+							filename = filename.replace(
+									STANDARD_LOG_FILE_EXTENSION,
+									'.' + t + STANDARD_LOG_FILE_EXTENSION)
+						else: # prepend timestamp
+							filename = t + '_' + filename
+					append_if_exist = not (self.log_mode == 'overwrite' or
+							self.log_mode == 'timestamp')
+					new_logfile = open(filename, 'a' if append_if_exist else 'w')
+					if self.logfile is not None:
+						text = u'Will now continue logging in log file ' + str(filename)
+						uid = str(uuid.uuid4())[0:8]
+						tim = time.time()
+						record = {
+							'uuid': uid, 
+							'time': tim,
+							'level': u'INFO',
+							'text': text,
+							'module': u'logger',
+							'function': u'LoggerComponent.open_logfile', 
+							'thread': '-', 
+							'logreceivedtime': tim}
+						self.logfile.write(str(record)+'\n')
+						self.logfile.close()
+					self.logfile = new_logfile
+					print('Logging to console and {filename} ...'.format(filename=filename))
+				except Exception as e:
+					print('Failed to open logfile {filename} for writing! Keeping previous configuration'.format(filename=filename))
+
+	def close_logfile(self):
+		if self.logfile is not None:
+			text = u'Closing of log file requested.'
+			uid = str(uuid.uuid4())[0:8]
+			tim = str(time.time())
+			record = {
+				'uuid': uid,
+				'time': tim,
+				'level': u'INFO',
+				'text': text,
+				'module': u'logger',
+				'function': u'LoggerComponent.close_logfile', 
+				'thread': u'-',
+				'logreceivedtime': tim}
+			self.logfile.write(str(record)+'\n')
+			self.logfile.close()
+			print('Closed current log file.')
+			self.logfile = None
+
+	def _logger_handle_iu_event(self, iu, event_type, own):
+		received_time = "%.3f" % time.time()
+		with LOGGER_LOCK:
+			try:
+				if iu.category == 'log':
+					pl = iu.payload
+					message = pl['text'] if 'text' in pl else '(No message.)'
+					uid = '????????' if 'uuid' not in pl else pl['uuid']
+					tim = '???' if 'time' not in pl else pl['time']
+					module = '???' if 'module' not in pl else pl['module']
+					function = '???' if 'function' not in pl else pl['function']
+					thread = '???' if 'thread' not in pl else pl['thread']
+					level = 'INFO' if 'level' not in pl else pl['level']
+					# dump to console
+					if level=='WARN':
+						level='WARNING'
+					if level not in ['DEBUG','INFO','WARNING','ERROR']:
+						level = 'INFO'
+					try:
+						print('%s %-8s {%s} {%s} {%s} %s'%(tim, ('['+level+']'), thread, module, function, message))
+					except:
+						print('Failed to format a string for printing!')
+					if self.logfile is not None:
+						try:
+							record = {
+								'uuid': uid, 
+								'time': tim, 
+								'level': level, 
+								'text': message,
+								'module': module,
+								'function': function, 
+								'thread': thread,
+								'logreceivedtime': received_time}
+							self.logfile.write(str(record) + '\n')
+						except:
+							print('Failed to write to logfile!')
+				elif iu.category == 'logcontrol':
+					cmd = iu.payload['cmd'] if 'cmd' in iu.payload else 'undef'
+					if cmd == 'open_log_file':
+						filename = iu.payload['filename'] if 'filename' in iu.payload else ''
+						if 'existing' in iu.payload:
+							log_mode_ = iu.payload['existing'].lower()
+							if log_mode_ not in LOG_MODES:
+								LOG_WARN(u'Value of "existing" should be "append", "timestamp", or "overwrite", continuing with mode {mode}'.format(mode=self.log_mode))
+							else:
+								self.log_mode = log_mode_
+						self.open_logfile(filename)
+					elif cmd == 'close_log_file':
+						self.close_logfile()
+					else:
+						LOG_WARN(u'Received unknown logcontrol command: '+str(cmd))
+			except Exception as e:
+				print('Exception while logging!')  # TODO write to file as well?
+				print(u'  '+str(traceback.format_exc()))

ipaacalib/python/src/ipaaca/util/notifier.py

@@ -4,7 +4,7 @@
 #  "Incremental Processing Architecture
 #   for Artificial Conversational Agents".
 #
-# Copyright (c) 2009-2014 Social Cognitive Systems Group
+# Copyright (c) 2009-2022 Social Cognitive Systems Group
 #                         CITEC, Bielefeld University
 #
 # http://opensource.cit-ec.de/projects/ipaaca/
@@ -33,9 +33,9 @@
 
 from __future__ import division, print_function
 
+import os
 import threading
 
-
 import ipaaca.buffer
 import ipaaca.iu
 import ipaaca.misc
@@ -64,11 +64,16 @@ class ComponentError(Exception):
 class ComponentNotifier(object):
 
 	NOTIFY_CATEGORY = "componentNotify"
+	CONTROL_CATEGORY = "componentControl"
 	SEND_CATEGORIES = "send_categories"
 	RECEIVE_CATEGORIES = "recv_categories"
+	CMD = "cmd"
 	STATE = "state"
 	NAME = "name"
+	WHO = "who" # list of names (or empty)
 	FUNCTION = "function"
+	PID = "pid"
+	CMD_REPORT = "report"
 
 	def __init__(self, component_name, component_function, send_categories, receive_categories, out_buffer=None, in_buffer=None):
 		self.component_name = component_name
@@ -116,14 +121,23 @@ class ComponentNotifier(object):
 			self.out_buffer.add(notify_iu)
 
 	def _handle_iu_event(self, iu, event_type, local):
-		if iu.payload[ComponentNotifier.NAME] == self.component_name:
-			return
-		with self.notification_handler_lock:
-			for h in self.notification_handlers:
-				h(iu, event_type, local)
-		if iu.payload[ComponentNotifier.STATE] == "new":
-			#print("submitting")
-			self._submit_notify(False)
+		if iu.category == ComponentNotifier.NOTIFY_CATEGORY:
+			if iu.payload[ComponentNotifier.NAME] == self.component_name:
+				return
+			with self.notification_handler_lock:
+				for h in self.notification_handlers:
+					h(iu, event_type, local)
+			if iu.payload[ComponentNotifier.STATE] == "new":
+				#print("submitting")
+				self._submit_notify(False)
+		elif iu.category == ComponentNotifier.CONTROL_CATEGORY:
+			cmd = iu.payload[ComponentNotifier.CMD]
+			if cmd=='report':
+				# Request to report (by component controller)
+				who = iu.payload[ComponentNotifier.WHO]
+				# If we are named specifically or it's a broadcast
+				if len(who)==0 or self.component_name in who:
+					self._submit_notify(False)
 
 	def add_notification_handler(self, handler):
 		with self.notification_handler_lock:
@@ -155,12 +169,14 @@ class ComponentNotifier(object):
 			if (not self.initialized):
 				self.timesync_slave = ipaaca.util.timesync.TimesyncSlave(component_name=self.component_name, timing_handler=self.launch_timesync_slave_handlers)
 				self.timesync_master = ipaaca.util.timesync.TimesyncMaster(component_name=self.component_name, timing_handler=self.launch_timesync_master_handlers)
-				self.in_buffer.register_handler(self._handle_iu_event, ipaaca.iu.IUEventType.MESSAGE, ComponentNotifier.NOTIFY_CATEGORY)
+				self.in_buffer.register_handler(self._handle_iu_event, ipaaca.iu.IUEventType.MESSAGE, [ComponentNotifier.NOTIFY_CATEGORY, ComponentNotifier.CONTROL_CATEGORY])
 				self._submit_notify(True)
 				self.initialized = True
 
 	def __enter__(self):
 		self.initialize()
+		return self
 
 	def __exit__(self, t, v, tb):
 		self.terminate()
+		return self

ipaacalib/python/src/ipaaca/util/timesync.py

@@ -4,7 +4,7 @@
 #  "Incremental Processing Architecture
 #   for Artificial Conversational Agents".
 #
-# Copyright (c) 2009-2014 Social Cognitive Systems Group
+# Copyright (c) 2009-2022 Social Cognitive Systems Group
 #                         CITEC, Bielefeld University
 #
 # http://opensource.cit-ec.de/projects/ipaaca/
@@ -32,6 +32,7 @@
 
 from __future__ import division, print_function
 
+import threading
 import time
 
 import ipaaca.buffer
@@ -45,7 +46,7 @@ class TimesyncMaster(object):
 		# component name to report (None => use buffer name)
 		self.component_name = component_name if component_name is not None else self.ob.unique_name
 		#
-		self.ob.register_handler(self.handle_timesync_master)
+		#self.ob.register_handler(self.handle_timesync_master)
 		self.ib.register_handler(self.handle_timesync_master)
 		# master_t1 is identical for all slaves
 		self.master_t1 = None
@@ -63,7 +64,7 @@ class TimesyncMaster(object):
 		self.timing_handler = timing_handler
 	
 	def send_master_timesync(self):
-		iu = ipaaca.iu.IU('timesyncRequest')
+		iu = ipaaca.iu.Message('timesyncRequest')
 		self.master_t1 = self.get_time()
 		iu.payload = {
 				'stage':'0',
@@ -71,47 +72,48 @@ class TimesyncMaster(object):
 				'master':self.component_name,
 				}
 		self.ob.add(iu)
-		print('Sent a stage 0 timesync as master '+self.component_name)
 	
 	def handle_timesync_master(self, iu, event_type, own):
 		master = iu.payload['master']
 		if not own and master == self.component_name:
-			if self.component_name == master:
-				# reply to our own initial IU
-				slave = iu.payload['slave']
-				stage = iu.payload['stage']
-				if stage=='1':
-					print('Received stage 1 from slave '+slave)
-					# initial reply by slave
-					t1 = iu.payload['slave_t1']
-					self.slave_t1s[slave] = float(t1)
-					t2 = self.master_t2s[slave] = self.get_time()
-					iu.payload.merge({'master_t2': str(t2), 'stage':'2'})
-					latency1 = t2 - self.master_t1
-					self.latencies[slave] = latency1
-					#print('Latency of round-trip 1: %.3f' % latency1)
-				elif stage=='3':
-					print('Received stage 3 from slave '+slave)
-					# second reply by slave
-					t2 = iu.payload['slave_t2']
-					self.slave_t2s[slave] = float(t2)
-					t_final = self.get_time()
-					latency1 = self.latencies[slave]
-					latency2 = t_final - self.master_t2s[slave]
-					latency = self.latencies[slave] = (latency1+latency2)/2.0
-					offset1 = (self.slave_t1s[slave]-self.master_t1)-latency/2.0
-					offset2 = (self.slave_t2s[slave]-self.master_t2s[slave])-latency/2.0
-					offset = (offset1+offset2)/2.0
-					iu.payload.merge({'stage':'4', 'latency': str(latency), 'offset':str(offset)})
-					if self.timing_handler is None:
-						print('Determined timing of timesync slave '+slave)
-						print('  Avg round-trip latency: %.3f s'%latency)
-						print('  Offset of their clock: %.3f s'%offset)
+			if event_type == ipaaca.IUEventType.ADDED or event_type == ipaaca.IUEventType.UPDATED:
+				if self.component_name == master:
+					# reply to our own initial IU
+					slave = iu.payload['slave']
+					stage = iu.payload['stage']
+					if stage=='1':
+						# initial reply by slave
+						t1 = iu.payload['slave_t1']
+						self.slave_t1s[slave] = float(t1)
+						t2 = self.master_t2s[slave] = self.get_time()
+						iu.payload.merge({'master_t2': str(t2), 'stage':'2'})
+						latency1 = t2 - self.master_t1
+						#print('Before stage 1 for '+slave+': '+str(self.latencies))
+						self.latencies[slave] = latency1
+						#print('After stage 1 for '+slave+': '+str(self.latencies))
+						#print('Latency of round-trip 1: %.3f' % latency1)
+					elif stage=='3':
+						#print('At stage 3 for '+slave+': '+str(self.latencies))
+						# second reply by slave
+						t2 = iu.payload['slave_t2']
+						self.slave_t2s[slave] = float(t2)
+						t_final = self.get_time()
+						latency1 = self.latencies[slave]
+						latency2 = t_final - self.master_t2s[slave]
+						latency = self.latencies[slave] = (latency1+latency2)/2.0
+						offset1 = (self.slave_t1s[slave]-self.master_t1)-latency/2.0
+						offset2 = (self.slave_t2s[slave]-self.master_t2s[slave])-latency/2.0
+						offset = (offset1+offset2)/2.0
+						iu.payload.merge({'stage':'4', 'latency': str(latency), 'offset':str(offset)})
+						if self.timing_handler is None:
+							print('Determined timing of timesync slave '+slave)
+							print('  Avg round-trip latency: %.3f s'%latency)
+							print('  Offset of their clock: %.3f s'%offset)
+						else:
+							self.timing_handler(self.component_name, slave, latency, offset)
 					else:
-						self.timing_handler(self.component_name, slave, latency, offset)
-				else:
-					# other stages are handled by time slave handler
-					pass
+						# other stages are handled by time slave handler
+						pass
 	
 	def get_time(self):
 		return time.time() + self.debug_offset
@@ -129,7 +131,6 @@ class TimesyncSlave(object):
 		#self.master_t2 = None
 		#self.master = None
 		self.latency = None
-		self.my_iu = None
 		#
 		self.debug_offset = debug_offset
 		#
@@ -142,30 +143,23 @@ class TimesyncSlave(object):
 		master = iu.payload['master']
 		stage = iu.payload['stage']
 		if self.component_name != master:
-			if not own:
+			if not own and stage=='0':
 				# reply only to IUs from others
-				if stage=='0':
-					#print('Received stage 0 from master '+master)
-					# initial reply to master
-					self.my_iu = ipaaca.iu.IU('timesyncReply')
-					# TODO: add grounded_in link too?
-					t1 = self.get_time()
-					self.my_iu.payload = iu.payload
-					self.my_iu.payload['slave'] = self.component_name
-					self.my_iu.payload['slave_t1'] = str(t1)
-					self.my_iu.payload['stage'] = '1'
-
-					#self.my_iu.payload.merge({
-					#		'slave':self.component_name,
-					#		'slave_t1':str(t1),
-					#		'stage':'1',
-					#		})
-					self.ob.add(self.my_iu)
-			else:
+				#print('Received stage 0 from master '+master)
+				# initial reply to master
+				myiu = ipaaca.iu.IU('timesyncReply')
+				# TODO: add grounded_in link too?
+				t1 = self.get_time()
+				myiu.payload = iu.payload
+				myiu.payload['slave'] = self.component_name
+				myiu.payload['slave_t1'] = str(t1)
+				myiu.payload['stage'] = '1'
+				self.ob.add(myiu)
+			elif iu.payload['slave'] == self.component_name:
 				if stage=='2':
 					#print('Received stage 2 from master '+master)
 					t2 = self.get_time()
-					self.my_iu.payload.merge({
+					iu.payload.merge({
 							'slave_t2':str(t2),
 							'stage':'3',
 							})

ipaacalib/python/src/listener.py

@@ -4,7 +4,7 @@
 #  "Incremental Processing Architecture
 #   for Artificial Conversational Agents".	
 #
-# Copyright (c) 2009-2013 Sociable Agents Group
+# Copyright (c) 2009-2022 Sociable Agents Group
 #                         CITEC, Bielefeld University	
 #
 # http://opensource.cit-ec.de/projects/ipaaca/
@@ -32,7 +32,6 @@
 
 
 import time
-import logging
 import ipaaca
 
 iu_to_write = None
@@ -69,8 +68,8 @@ while True:
 			else:
 				iu.payload = {'a': 'reset'}
 			
-		except ipaaca.IUUpdateFailedError, e:
-			ipaaca.logger.warning("Payload update failed (IU changed in the mean time)")
+		except ipaaca.IUUpdateFailedError as e:
+			print("Payload update failed (IU changed in the mean time)")
 	time.sleep(0.1)
 
 exit(0)

ipaacalib/python/src/run.py

@@ -2,5 +2,5 @@
 
 import ipaaca
 
-print "{this is the IpaacaPython run.py doing nothing at all}"
+print("{this is the IpaacaPython run.py doing nothing at all}")
 

ipaacalib/python/test/src/ipaacaspam.py

@@ -2,7 +2,7 @@
 #  "Incremental Processing Architecture
 #   for Artificial Conversational Agents".	
 #
-# Copyright (c) 2009-2013 Sociable Agents Group
+# Copyright (c) 2009-2022 Sociable Agents Group
 #                         CITEC, Bielefeld University	
 #
 # http://opensource.cit-ec.de/projects/ipaaca/

ipaacalib/python/test/src/testipaaca.py

@@ -2,7 +2,7 @@
 #  "Incremental Processing Architecture
 #   for Artificial Conversational Agents".	
 #
-# Copyright (c) 2009-2013 Sociable Agents Group
+# Copyright (c) 2009-2022 Sociable Agents Group
 #                         CITEC, Bielefeld University	
 #
 # http://opensource.cit-ec.de/projects/ipaaca/

ipaacalib/python/test/src/testnotifier.py

@@ -2,7 +2,7 @@
 #  "Incremental Processing Architecture
 #   for Artificial Conversational Agents".  
 #
-# Copyright (c) 2009-2013 Sociable Agents Group
+# Copyright (c) 2009-2022 Sociable Agents Group
 #                         CITEC, Bielefeld University   
 #
 # http://opensource.cit-ec.de/projects/ipaaca/

ipaacalib/python/test/src/textsender.py

@@ -2,7 +2,7 @@
 #  "Incremental Processing Architecture
 #   for Artificial Conversational Agents".	
 #
-# Copyright (c) 2009-2013 Sociable Agents Group
+# Copyright (c) 2009-2022 Sociable Agents Group
 #                         CITEC, Bielefeld University	
 #
 # http://opensource.cit-ec.de/projects/ipaaca/

ipaacalib/python/test/src/wordsender.py

@@ -2,7 +2,7 @@
 #  "Incremental Processing Architecture
 #   for Artificial Conversational Agents".	
 #
-# Copyright (c) 2009-2013 Sociable Agents Group
+# Copyright (c) 2009-2022 Sociable Agents Group
 #                         CITEC, Bielefeld University	
 #
 # http://opensource.cit-ec.de/projects/ipaaca/

ipaacatools/scripts/ipaaca-catlog

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# This file is part of IPAACA, the
+#  "Incremental Processing Architecture
+#   for Artificial Conversational Agents".
+#
+# Copyright (c) 2009-2015 Social Cognitive Systems Group
+#                         CITEC, Bielefeld University
+#
+# http://opensource.cit-ec.de/projects/ipaaca/
+# http://purl.org/net/ipaaca
+#
+# This file may be licensed under the terms of of the
+# GNU Lesser General Public License Version 3 (the ``LGPL''),
+# or (at your option) any later version.
+#
+# Software distributed under the License is distributed
+# on an ``AS IS'' basis, WITHOUT WARRANTY OF ANY KIND, either
+# express or implied. See the LGPL for the specific language
+# governing rights and limitations.
+#
+# You should have received a copy of the LGPL along with this
+# program. If not, go to http://www.gnu.org/licenses/lgpl.html
+# or write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# The development of this software was supported by the
+# Excellence Cluster EXC 277 Cognitive Interaction Technology.
+# The Excellence Cluster EXC 277 is a grant of the Deutsche
+# Forschungsgemeinschaft (DFG) in the context of the German
+# Excellence Initiative.
+
+
+from __future__ import division, print_function
+
+import sys
+import ipaaca
+import traceback
+import io
+
+def pretty_printed_time(t):
+	if t<0:
+		t = -t
+		sign = '-'
+	else:
+		sign = ' '
+	s = float(t)
+	h = int(s/3600)
+	m = int(s/60) - h*60
+	s -= h*3600 + m*60
+	ms = int((s-int(s))*1000)
+	return sign+'%01d:%02d:%02d.%03d'%(h, m, int(s), ms)
+
+REPLACEMENT_COLORS={
+			'30': 'black',
+			'31': 'red',
+			'32': 'green',
+			'33': 'cyan',
+			'34': 'blue',
+			'35': 'brown',
+			'36': 'magenta',
+			'37': 'grey',
+			'248': 'lightgray',
+			# should add more colors
+		}
+
+def replace_ansi_html(s):
+	r = u''
+	in_esc = False
+	last_color = u''
+	#skip_one_double_wide_bar = False
+	for c in s:
+		if in_esc:
+			if c=='m':
+				in_esc = False
+				itms = last_color.replace('[','').replace(']','').split(';')
+				col = None
+				bold = False
+				if itms[0]=='':
+					r += '</code></font><code>'
+				else:
+					for i in itms:
+						if i=='1':
+							bold = True
+						elif i in REPLACEMENT_COLORS:
+							col = REPLACEMENT_COLORS[i]
+					if col is None:
+						if bold:
+							col = 'grey'
+						else:
+							col = 'black'
+							print('Warning: unknown colors '+str(itms))
+					r += '</code><font color="'+col+'"><code>'
+			else:
+				last_color += c
+		else:
+			#if c in u'▁▂▃▄▅▆▇█':
+			#	if skip_one_double_wide_bar:
+			#		skip_one_double_wide_bar = False
+			#	else:
+			#		r += c
+			#		skip_one_double_wide_bar = True
+			#el
+			if c=='':
+				in_esc = True
+				last_color = ''
+			else:
+				r += c
+	return r
+
+def print_header(formt, fields):
+	s = u''
+	if formt=='html':
+		s += '<html><head><meta charset="UTF-8"><title>flexdiam log view</title></head>\n<body>\n<table width="100%" border="1" bordercolor="lightgray" style="white-space:pre">\n<tr>'
+		for f in fields:
+			s+='<th>'+f+'</th>'
+		s += '</tr>'
+	return s
+
+def print_footer(formt):
+	s = u''
+	if formt=='html':
+		s += '</tr>\n</body>\n</html>'
+	return s
+
+def print_record(data, delimiter, formt):
+	if formt=='html':
+		s = u'<tr>'
+		for d in data:
+			d2 = d.replace('<', '&lt;').replace('>', '&gt;').replace('\n', '<br/>').replace('"', '&quot;')
+			d3 = replace_ansi_html(d2)
+			#s += u'<td><code>' + d2.replace('<', '&lt;').replace('>', '&gt;').replace('\n', '<br/>') + u'</code></td>'
+			s += u'<td><code>' + d3 + u'</code></td>'
+		s += u'</tr>'
+		return s
+	else:
+		return delimiter.join(data)
+
+def modify(key, value, strip=False, pretty_printed_times=False, time_offset=0.0):
+	if key=='time':
+		f = float(value.strip()) - time_offset
+		return pretty_printed_time(f) if pretty_printed_times else str(f)
+	else:
+		return value.strip() if strip else value
+
+if __name__=='__main__':
+	try:
+		iap = ipaaca.IpaacaArgumentParser('ipaaca-logcat')
+		iap.add_argument(
+			'-s', '--strip-fields',
+			dest='strip', action='store_true',
+			default=False,
+			help='Strip leading/trailing whitespace from all fields')
+		iap.add_argument(
+			'-p', '--pretty-printed-times',
+			dest='pretty_printed_times', action='store_true',
+			default=False,
+			help='Print human-readable times (hh:mm:ss.ms) instead of float seconds')
+		iap.add_argument(
+			'--format',
+			dest='format', nargs=1,
+			default=['plain'],
+			help='output format (plain, html) (default plain)')
+		iap.add_argument(
+			'-o', '--output-file',
+			dest='output_file', nargs=1,
+			default=['-'],
+			help='output file name (default \'-\' -> standard terminal output)')
+		iap.add_argument(
+			'-d', '--delimiter',
+			dest='delimiter', nargs=1,
+			default=['\t'],
+			help='field delimiter, interpreted as python unicode string (default \'\\t\')')
+		iap.add_argument(
+			'-t', '--align-times',
+			dest='align_times', nargs=2,
+			default=['0', '0'],
+			help='Calculate relative output timestamps (default: 0  0 => keep)\nFirst argument is a reference event timestamp from the log file\nSecond argument is the new output time for that same event')
+		iap.add_argument(
+			'-f', '--fields',
+			dest='fields', default=['time', 'text'], nargs='+',
+			help='fields to print (defaults: \'time\' \'text\')')
+		arguments = iap.parse_args()
+		delimiter = eval("u'"+arguments.delimiter[0]+"'", {"__builtins__":None}, {} )
+		ref_t, out_t = float(arguments.align_times[0]), float(arguments.align_times[1])
+		time_offset = ref_t - out_t
+		#print(arguments); sys.exit(1)
+		#modify = (lambda s: s.strip()) if arguments.strip else (lambda s: s)
+		#modify = lambda s: type(s).__name__
+		fil = sys.stdout
+		if arguments.output_file[0] != '-':
+			fil = io.open(arguments.output_file[0], 'w', encoding='utf8')
+		fil.write(print_header(arguments.format[0], arguments.fields)+'\n')
+		for line in sys.stdin:
+			record = eval(line.strip(), {"__builtins__":None}, {} )
+			data = [modify(f, unicode(record[f]), arguments.strip, arguments.pretty_printed_times, time_offset) for f in arguments.fields]
+			u = print_record(data, delimiter, arguments.format[0])
+			res = u'{}'.format(u) #.decode('utf-8')
+			fil.write(u''+res+'\n' )
+		fil.write(print_footer(arguments.format[0])+'\n')
+		if fil != sys.stdout: fil.close()
+	except (KeyboardInterrupt, SystemExit):
+		pass # ret below
+	except Exception, e:
+		print(u'Exception: '+unicode(traceback.format_exc()))
+		ipaaca.exit(1)
+	ipaaca.exit(0)
+

ipaacatools/scripts/ipaaca-iu-injector

@@ -31,7 +31,7 @@
 # Forschungsgemeinschaft (DFG) in the context of the German
 # Excellence Initiative.
 
-from __future__ import division, print_function
+#from __future__ import division, print_function
 
 import itertools
 import os
@@ -44,7 +44,7 @@ import ipaaca
 
 def iu_update_handler(iu, event_type, local):
 	try:
-		print(event_type + ': ' + unicode(iu))
+		print(event_type + ': ' + str(iu))
 	except:
 		print(u"An error occurred printing an IU for an event of type "+event_type)
 
@@ -86,14 +86,16 @@ parser.add_argument(
 if __name__ == '__main__':
 	arguments = parser.parse_args()
 
+	print('BackEnd is '+str(ipaaca.backend.get_default_backend().name))
+
 	ob = ipaaca.OutputBuffer('IpaacaIUInjector')
 	ob.register_handler(iu_update_handler)
 	iu = ipaaca.Message(arguments.category) if arguments.iu_type == 'Message' else ipaaca.IU(arguments.category)
 	if arguments.json_payload:
 		# Treat payload values as Python expressions
-		iu.payload = {k: eval(v) for (k, v) in itertools.izip_longest(arguments.payload[::2],	arguments.payload[1::2])}
+		iu.payload = {k: eval(v) for (k, v) in itertools.zip_longest(arguments.payload[::2],	arguments.payload[1::2])}
 	else:
-		iu.payload = {k: v for (k, v) in itertools.izip_longest(arguments.payload[::2],	arguments.payload[1::2])}
+		iu.payload = {k: v for (k, v) in itertools.zip_longest(arguments.payload[::2],	arguments.payload[1::2])}
 	ob.add(iu)
 
 	print(
@@ -112,8 +114,7 @@ if __name__ == '__main__':
 			time.sleep(0.1)
 	except KeyboardInterrupt:
 		pass
-
-	if platform.system() == 'Windows':
-		os._exit(0)
-	else:
-		sys.exit(0)
+	except Exception as e:
+		print(u'Exception: '+str(traceback.format_exc()))
+		ipaaca.exit(1)
+	ipaaca.exit(0)

ipaacatools/scripts/ipaaca-iu-sniffer

@@ -31,7 +31,7 @@
 # Forschungsgemeinschaft (DFG) in the context of the German
 # Excellence Initiative.
 
-from __future__ import division, print_function
+#from __future__ import division, print_function
 
 import logging
 import os
@@ -51,7 +51,7 @@ def event_type_color(typ):
 		'MESSAGE': '34;1',
 		'COMMITTED': '35;1',
 		'LINKSUPDATED': '36;1',
-		'RETRACTED': '37;1',
+		'RETRACTED': '37',
 	}
 	return colors.get(typ, '1')
 
@@ -61,13 +61,13 @@ def highlight_if_color(s, c='1'):
 def pretty_printed_dict(d):
 	s='{\n'
 	for k, v in d.items():
-		if isinstance(v, unicode) or isinstance(v, str):
-			v = "'"+unicode(v)+"'"
+		if isinstance(v, str) or isinstance(v, str):
+			v = "'"+str(v)+"'"
 		else:
-			v = unicode(v)
+			v = str(v)
 		v2 = v if len(v) <= arguments.max_size else v[:arguments.max_size] + '<excess length omitted>'
 		v2.replace('\\','\\\\').replace('\n', highlight_if_color('\\n'))
-		s += "\t '%s': %s,\n" % (highlight_if_color(unicode(k),'1'), unicode(v2))
+		s += "\t '%s': %s,\n" % (highlight_if_color(str(k),'1'), str(v2))
 	s+='}'
 	return s
 
@@ -97,10 +97,11 @@ def iu_event_handler(iu, event_type, local):
 	print(pretty_printed_iu_event(iu, event_type, local), end='\n\n')
 
 def exit(code):
-	if platform.system() == 'Windows':
-		os._exit(code)
-	else:
-		sys.exit(code)
+	ipaaca.exit(code)
+	#if platform.system() == 'Windows':
+	#	os._exit(code)
+	#else:
+	#	sys.exit(code)
 
 parser = ipaaca.IpaacaArgumentParser(description='Ipaaca IU Sniffer -- Selectively listen to IPAACA traffic')
 parser.add_argument(
@@ -123,7 +124,7 @@ parser.add_argument(
 	dest='event_types',
 	metavar='EVENT-TYPE',
 	nargs='+',
-	help='set event types (ADDED, MESSAGE, UPDATED, LINKSUPDATED, COMMITTED)')
+	help='set event types ({})'.format(' '.join(ipaaca.iu.IUEventType._choices)))
 parser.add_argument(
 	'-r', '--regex',
 	action='store_true',
@@ -147,17 +148,30 @@ if __name__ == '__main__':
 	arguments = parser.parse_args()
 	buffers = {}
 
+	backend_name = str(ipaaca.backend.get_default_backend().name)
+	print('BackEnd is '+backend_name)
+	universal_listener_category = ''
+	if backend_name == 'mqtt':
+		universal_listener_category = '#'
+		if arguments.categories == ['']: arguments.categories = ['#']
+	elif backend_name == 'ros':
+		if arguments.categories == [''] or arguments.regex:
+			print('ATTENTION: listening to all categories not implemented for ROS back-end!')
+			print('  (By extension, the same goes for filtering all categories by regex.)')
+			print('  !! You will receive nothing, please provide fixed category names. !!')
+
+
 	# Create one input buffer for each channel we are listening on
 	for channel in arguments.channels:
 		buffers[channel] = ipaaca.InputBuffer(
 			'IpaacaIUSniffer',
-			category_interests=arguments.categories if not arguments.regex else [''],
+			category_interests=arguments.categories if not arguments.regex else [universal_listener_category],
 			channel=channel,
 			resend_active=True)
 		# Check whether the specified event_types are valid
 		if arguments.event_types is not None:
 			for et in arguments.event_types:
-				if et.upper() not in ['ADDED', 'MESSAGE', 'UPDATED', 'LINKSUPDATED', 'COMMITTED']:
+				if et.upper() not in ipaaca.iu.IUEventType._choices:
 					print('ERROR: "{et}" is not a valid IPAACA event type.'.format(et=et.upper()))
 					exit(code=1)
 		buffers[channel].register_handler(
@@ -173,7 +187,7 @@ if __name__ == '__main__':
 	else:
 		print(', '.join([highlight_if_color(et.upper(), event_type_color(et.upper())) for et in arguments.event_types]))
 	print('  * for category/ies', end='')
-	if arguments.categories == ['']:
+	if arguments.categories == [universal_listener_category]:
 		print(': any')
 	else:
 		if arguments.regex:
@@ -186,4 +200,7 @@ if __name__ == '__main__':
 			time.sleep(1)
 	except KeyboardInterrupt:
 		pass
-	exit(code=0)
+	except Exception as e:
+		print(u'Exception: '+str(traceback.format_exc()))
+		ipaaca.exit(1)
+	ipaaca.exit(0)

ipaacatools/scripts/ipaaca-logger

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# This file is part of IPAACA, the
+#  "Incremental Processing Architecture
+#   for Artificial Conversational Agents".
+#
+# Copyright (c) 2009-2015 Social Cognitive Systems Group
+#                         CITEC, Bielefeld University
+#
+# http://opensource.cit-ec.de/projects/ipaaca/
+# http://purl.org/net/ipaaca
+#
+# This file may be licensed under the terms of of the
+# GNU Lesser General Public License Version 3 (the ``LGPL''),
+# or (at your option) any later version.
+#
+# Software distributed under the License is distributed
+# on an ``AS IS'' basis, WITHOUT WARRANTY OF ANY KIND, either
+# express or implied. See the LGPL for the specific language
+# governing rights and limitations.
+#
+# You should have received a copy of the LGPL along with this
+# program. If not, go to http://www.gnu.org/licenses/lgpl.html
+# or write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# The development of this software was supported by the
+# Excellence Cluster EXC 277 Cognitive Interaction Technology.
+# The Excellence Cluster EXC 277 is a grant of the Deutsche
+# Forschungsgemeinschaft (DFG) in the context of the German
+# Excellence Initiative.
+
+from __future__ import division, print_function
+
+import sys
+import time
+
+import ipaaca
+import ipaaca.util.logger as ipaacalog
+
+
+def main(log_mode, filename=None):
+	ipaacalog.logger_send_ipaaca_logs(False)
+	il = ipaacalog.LoggerComponent(filename, log_mode)
+	try:
+		while True:
+			time.sleep(1)
+	except KeyboardInterrupt:
+		il.close_logfile()
+		print('Logging-Component closed by keyboard interrupt.')
+
+
+if __name__ == '__main__':
+	import traceback
+	try:
+		iap = ipaaca.IpaacaArgumentParser(
+			'ipaaca-logger')
+		iap.add_argument(
+			'-m', '--log-mode', dest='log_mode',
+			choices=ipaacalog.LOG_MODES,
+			default='append',
+			help="set what to do when logfile exists "
+					"(default: 'append'; 'timestamp' adds timestamp in any case)")
+		iap.add_argument(
+			'filename', nargs='?',
+			metavar='FILE',
+			help='set name of logfile')
+		arguments = iap.parse_args()
+		main(arguments.log_mode, arguments.filename)
+	except KeyboardInterrupt:
+		pass
+	except Exception, e:
+		print(u'Exception: '+unicode(traceback.format_exc()))
+		ipaaca.exit(1)
+	ipaaca.exit(0)
+

ipaacatools/scripts/ipaaca-ping

+#!/usr/bin/env python
+
+import ipaaca
+import time
+
+class PingSender(object):
+	def __init__(self):
+		self.pings = []
+		self.times = []
+		self.ob = ipaaca.OutputBuffer('IpaacaPing')
+		self.ob.register_handler(self.iu_event_handler)
+	
+	def iu_event_handler(self, iu, event_type, local):
+		recv_t = time.time()
+		if event_type=='UPDATED':
+			print('OK')
+			send_t = float(iu.payload['sender_local_t'])
+			receiver_recv_t = float(iu.payload['receiver_local_t'])
+			round_trip_t = recv_t - send_t
+			locally_estimated_receiver_recv_t = (recv_t + send_t) / 2
+			estimated_clock_skew = receiver_recv_t - locally_estimated_receiver_recv_t
+			self.times.append(estimated_clock_skew)
+			average_clock_difference = sum(self.times) / len(self.times)
+			self.pings.append(500.0 * round_trip_t)
+			average_estimated_single_trip = sum(self.pings) / len(self.pings)
+			print('Received ping reply')
+			print('  measured round trip time [ms]: %0.3f'%(1000.0 * round_trip_t))
+			print('  estimated single trip [ms]: %0.3f'%(500.0 * round_trip_t))
+			print('  averaged single trip [ms]: %0.3f'%(average_estimated_single_trip))
+			print('  estimated system clock difference [s]: %0.3f'%(average_clock_difference))
+
+	def send_ping(self):
+		t = time.time()
+		iu = ipaaca.IU('ipaacaPing')
+		iu.payload['sender_local_t'] = t
+		self.ob.add(iu)
+
+
+iap = ipaaca.IpaacaArgumentParser('ipaaca-ping')
+arguments = iap.parse_args()
+
+ps = PingSender()
+while True:
+	ps.send_ping()
+	time.sleep(1)
+
+

ipaacatools/scripts/ipaaca-pong

+#!/usr/bin/env python
+
+import ipaaca
+import time
+
+def iu_event_handler(iu, event_type, local):
+	if event_type=='ADDED':
+		iu.payload['receiver_local_t'] = time.time()
+		print('Sent IPAACA ping reply')
+
+iap = ipaaca.IpaacaArgumentParser('ipaaca-pong')
+arguments = iap.parse_args()
+
+ib = ipaaca.InputBuffer('IpaacaPong', ['ipaacaPing'])
+ib.register_handler(iu_event_handler)
+
+while True:
+	time.sleep(1)
+

ipaacatools/scripts/ipaaca-session-record

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# This file is part of IPAACA, the
+#  "Incremental Processing Architecture
+#   for Artificial Conversational Agents".
+#
+# Copyright (c) 2009-2015 Social Cognitive Systems Group
+#                         CITEC, Bielefeld University
+#
+# http://opensource.cit-ec.de/projects/ipaaca/
+# http://purl.org/net/ipaaca
+#
+# This file may be licensed under the terms of of the
+# GNU Lesser General Public License Version 3 (the ``LGPL''),
+# or (at your option) any later version.
+#
+# Software distributed under the License is distributed
+# on an ``AS IS'' basis, WITHOUT WARRANTY OF ANY KIND, either
+# express or implied. See the LGPL for the specific language
+# governing rights and limitations.
+#
+# You should have received a copy of the LGPL along with this
+# program. If not, go to http://www.gnu.org/licenses/lgpl.html
+# or write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# The development of this software was supported by the
+# Excellence Cluster EXC 277 Cognitive Interaction Technology.
+# The Excellence Cluster EXC 277 is a grant of the Deutsche
+# Forschungsgemeinschaft (DFG) in the context of the German
+# Excellence Initiative.
+
+from __future__ import division, print_function
+
+import logging
+import os
+import platform
+import re
+import sys
+import time
+
+import ipaaca
+
+
+def event_type_color(typ):
+	colors = {
+		'ADDED': '32;1',
+		'RETRACTED': '31;1',
+		'UPDATED': '33;1',
+		'MESSAGE': '34;1',
+		'COMMITTED': '35;1',
+		'LINKSUPDATED': '36;1',
+		'RETRACTED': '37;1',
+	}
+	return colors.get(typ, '1')
+
+def highlight_if_color(s, c='1'):
+	return s if arguments.color else '[' + c + 'm' + s +''
+
+def pretty_printed_dict(d):
+	s='{\n'
+	for k, v in d.items():
+		if isinstance(v, unicode) or isinstance(v, str):
+			v = "'"+unicode(v)+"'"
+		else:
+			v = unicode(v)
+		v2 = v if len(v) <= arguments.max_size else v[:arguments.max_size] + '<excess length omitted>'
+		v2.replace('\\','\\\\').replace('\n', highlight_if_color('\\n'))
+		s += "\t '%s': %s,\n" % (highlight_if_color(unicode(k),'1'), unicode(v2))
+	s+='}'
+	return s
+
+def pretty_printed_iu_event(iu, event_type, local):
+	s = ''
+	t = time.time()
+	lt = time.localtime(t)
+	s += highlight_if_color('%.3f' % t, '1')
+	s += ' %02d:%02d:%02d' % (lt.tm_hour, lt.tm_min, lt.tm_sec)
+	s += ' ' + highlight_if_color('%-9s' % event_type, event_type_color(event_type))
+	s += ' category=' + highlight_if_color(iu.category,event_type_color(event_type))
+	s += ' channel=' + iu.buffer._channel
+	s += ' uid=' + iu.uid
+	s += ' owner=' + iu.owner_name
+	if event_type is not 'MESSAGE':
+		s += '\nlinks=' + pretty_printed_dict(iu.get_all_links())
+	s += '\npayload=' + pretty_printed_dict(iu.payload)
+	return s
+
+def machine_formatted_iu_event(iu, event_type, local):
+	return repr({
+				'time': time.time(),
+				'uid': iu.uid,
+				'category': iu.category,
+				'type': event_type,
+				'payload': iu.payload,
+				#'links': iu.links,
+			}).encode('utf-8')
+
+def iu_event_recorder_handler(logfile, iu, event_type, local):
+	if arguments.regex:
+		for cat in arguments.categories: # actually now regexs, not categories
+			if re.match(cat, iu.category):
+				break
+		else:
+			return
+	if logfile is None:
+		# requested log to stdout, only printing the structured data
+		print(machine_formatted_iu_event(iu, event_type, local)+'\n')
+	else:
+		# write structured data to log file
+		logfile.write(machine_formatted_iu_event(iu, event_type, local)+'\n')
+		# duplicate human-readable on terminal (unless data being sent there)
+		print(pretty_printed_iu_event(iu, event_type, local), end='\n\n')
+
+def exit(code):
+	ipaaca.exit(code)
+	#if platform.system() == 'Windows':
+	#	os._exit(code)
+	#else:
+	#	sys.exit(code)
+
+parser = ipaaca.IpaacaArgumentParser(description='Ipaaca IU Session Recorder -- Selectively record [and print] IU streams')
+parser.add_argument(
+	'-o', '--output-file',
+	dest='output_file',
+	default=['default.ipaacasession'],
+	metavar='OUTPUT-FILE',
+	nargs=1,
+	help="set the channels to listen on (otherwise 'default')")
+parser.add_argument(
+	'--channels',
+	dest='channels',
+	default=['default'],
+	metavar='CHANNEL',
+	nargs='+',
+	help="set the channels to listen on (otherwise 'default')")
+parser.add_argument(
+	'-c', '--categories',
+	default=[''],
+	dest='categories',
+	metavar='CATEGORY',
+	nargs='+',
+	help='set categories (or regex patterns) to be matched')
+parser.add_argument(
+	'-e', '--event-types',
+	default=None,
+	dest='event_types',
+	metavar='EVENT-TYPE',
+	nargs='+',
+	help='set event types ({})'.format(' '.join(ipaaca.iu.IUEventType._choices)))
+parser.add_argument(
+	'-r', '--regex',
+	action='store_true',
+	dest='regex',
+	help='match categories by regular expressions')
+parser.add_argument(
+	'--no-color',
+	action='store_true',
+	dest='color',
+	help='do not colorize output')
+parser.add_argument(
+	'--size-limit',
+	default=2048,
+	dest='max_size',
+	metavar='LIMIT',
+	type=int,
+	help='limit payload display chars (default: 2048)')
+
+
+if __name__ == '__main__':
+	arguments = parser.parse_args()
+	buffers = {}
+
+	log_file = None if (arguments.output_file[0]=='-') else open(arguments.output_file[0], 'w')
+	
+	# Create one input buffer for each channel we are listening on
+	for channel in arguments.channels:
+		buffers[channel] = ipaaca.InputBuffer(
+			'IpaacaIUSniffer',
+			category_interests=arguments.categories if not arguments.regex else [''],
+			channel=channel,
+			resend_active=True)
+		# Check whether the specified event_types are valid
+		if arguments.event_types is not None:
+			for et in arguments.event_types:
+				if et.upper() not in ipaaca.iu.IUEventType._choices:
+					print('ERROR: "{et}" is not a valid IPAACA event type.'.format(et=et.upper()))
+					exit(code=1)
+		buffers[channel].register_handler(
+			lambda iu,typ,loc: iu_event_recorder_handler(log_file, iu, typ, loc),
+			for_event_types=[et.upper() for et in arguments.event_types] if arguments.event_types is not None else None)
+
+	# Tell user what we are doing
+	print('ipaaca-session-record listening')
+	print('  * writing to file: '+('(stdout)' if log_file is None else arguments.output_file[0]))
+	print('  * on channel(s): ' + ', '.join(arguments.channels))
+	print('  * for event type(s): ', end='')
+	if arguments.event_types is None:
+		print('any')
+	else:
+		print(', '.join([highlight_if_color(et.upper(), event_type_color(et.upper())) for et in arguments.event_types]))
+	print('  * for category/ies', end='')
+	if arguments.categories == ['']:
+		print(': any')
+	else:
+		if arguments.regex:
+			print(' that match the regex', end='')
+		print(': ' + ', '.join(arguments.categories))
+
+	# Wait for the user to kill the sniffer
+	try:
+		while True:
+			time.sleep(1)
+	except KeyboardInterrupt:
+		pass
+	except Exception, e:
+		print(u'Exception: '+unicode(traceback.format_exc()))
+		if log_file is not None:
+			log_file.close()
+		ipaaca.exit(1)
+	if log_file is not None:
+		log_file.close()
+	ipaaca.exit(0)
+

ipaacatools/scripts/ipaaca-session-replay

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# This file is part of IPAACA, the
+#  "Incremental Processing Architecture
+#   for Artificial Conversational Agents".
+#
+# Copyright (c) 2009-2015 Social Cognitive Systems Group
+#                         CITEC, Bielefeld University
+#
+# http://opensource.cit-ec.de/projects/ipaaca/
+# http://purl.org/net/ipaaca
+#
+# This file may be licensed under the terms of of the
+# GNU Lesser General Public License Version 3 (the ``LGPL''),
+# or (at your option) any later version.
+#
+# Software distributed under the License is distributed
+# on an ``AS IS'' basis, WITHOUT WARRANTY OF ANY KIND, either
+# express or implied. See the LGPL for the specific language
+# governing rights and limitations.
+#
+# You should have received a copy of the LGPL along with this
+# program. If not, go to http://www.gnu.org/licenses/lgpl.html
+# or write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# The development of this software was supported by the
+# Excellence Cluster EXC 277 Cognitive Interaction Technology.
+# The Excellence Cluster EXC 277 is a grant of the Deutsche
+# Forschungsgemeinschaft (DFG) in the context of the German
+# Excellence Initiative.
+
+from __future__ import division, print_function
+
+import itertools
+import os
+import platform
+import sys
+import time
+import traceback
+
+import ipaaca
+
+
+def iu_update_handler(iu, event_type, local):
+	try:
+		print(event_type + ': ' + unicode(iu))
+	except:
+		print(u"An error occurred printing an IU for an event of type "+event_type)
+
+
+parser = ipaaca.IpaacaArgumentParser(description='Ipaaca Session Replay -- Replay IU event logs from the command line')
+parser.add_argument(
+	'-i', '--input-file',
+	default=['default.ipaacasession'],
+	dest='input_file',
+	metavar='INPUT-FILE',
+	nargs=1,
+	help='file from which to read IU data for replay')
+parser.add_argument(
+	'-k', '--keep-alive',
+	default=1.0,
+	dest='keep_alive',
+	metavar='SECONDS',
+	type=float,
+	help='set time in seconds to wait for final remote IU updates (default: 1.0)')
+parser.add_argument(
+	'-c', '--categories',
+	dest='categories',
+	metavar='CATEGORIES',
+	required=False,
+	default=[],
+	nargs='*',
+	help='send events for these categories only')
+
+
+if __name__ == '__main__':
+	arguments = parser.parse_args()
+	
+	for_categories = list(arguments.categories)
+	
+	ob = ipaaca.OutputBuffer('IpaacaSessionReplay')
+	# CAUTION: special semantics for replay script: do NOT auto-retract at shutdown
+	#   (should only replicate observed retractions form the data file!)
+	ob._teardown = lambda: True
+	ob.register_handler(iu_update_handler)
+
+	known_ius = {}
+	
+	log_file = sys.stdin if (arguments.input_file[0]=='-') else open(arguments.input_file[0], 'r')
+	ref_time = None
+	last_time = None
+	for line in log_file:
+		record = eval(line, {'__builtins__': None}, {})
+		# event data
+		uid = record['uid']
+		category = record['category']
+		typ = record['type']
+		payload = record['payload']
+		# consider only if category filter passes
+		if len(for_categories)==0 or category in for_categories:
+			# take ref time from first event
+			if ref_time is None:
+				ref_time = record['time']
+				last_time = ref_time
+			# recreate delay from previous event
+			print('Sleep for '+str(record['time'] - last_time)+' s')
+			time.sleep(record['time'] - last_time)
+			last_time = record['time']
+			# do it
+			print('Synthesize event '+typ+' for IU UID '+uid+', category "'+category+'"')
+			if typ=='MESSAGE':
+				msg = ipaaca.Message(category)
+				msg.payload = payload
+				ob.add(msg)
+			elif typ=='ADDED':
+				if uid in known_ius:
+					print('ERROR - already added UID '+uid+' before!')
+				else:
+					iu = ipaaca.IU(category)
+					iu.payload = payload
+					ob.add(iu)
+					known_ius[uid] = iu
+			elif typ=='UPDATED':
+				if uid not in known_ius:
+					print('ERROR - have not received IU with UID '+uid+' before!')
+				else:
+					iu = known_ius[uid]
+					iu.payload = payload
+			elif typ=='COMMITTED':
+				if uid not in known_ius:
+					print('ERROR - have not received IU with UID '+uid+' before!')
+				else:
+					iu = known_ius[uid]
+					iu.commit()
+			elif typ=='RETRACTED':
+				if uid not in known_ius:
+					print('ERROR - have not received IU with UID '+uid+' before!')
+				else:
+					iu = known_ius[uid]
+					ob.remove(iu)
+	
+	try:
+		print('Waiting; specified grace time '+str(arguments.keep_alive)+' s')
+		time.sleep(arguments.keep_alive)
+	except KeyboardInterrupt:
+		pass
+	except Exception, e:
+		print(u'Exception: '+unicode(traceback.format_exc()))
+		if log_file is not sys.stdin:
+			log_file.close()
+		ipaaca.exit(1)
+	if log_file is not sys.stdin:
+		log_file.close()
+	ipaaca.exit(0)
+

ipaacatools/scripts/ipaaca-socket-hub

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# This file is part of IPAACA, the
+#  "Incremental Processing Architecture
+#   for Artificial Conversational Agents".
+#
+# Copyright (c) 2009-2016 Social Cognitive Systems Group
+#                         CITEC, Bielefeld University
+#
+# http://opensource.cit-ec.de/projects/ipaaca/
+# http://purl.org/net/ipaaca
+#
+# This file may be licensed under the terms of of the
+# GNU Lesser General Public License Version 3 (the ``LGPL''),
+# or (at your option) any later version.
+#
+# Software distributed under the License is distributed
+# on an ``AS IS'' basis, WITHOUT WARRANTY OF ANY KIND, either
+# express or implied. See the LGPL for the specific language
+# governing rights and limitations.
+#
+# You should have received a copy of the LGPL along with this
+# program. If not, go to http://www.gnu.org/licenses/lgpl.html
+# or write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# The development of this software was supported by the
+# Excellence Cluster EXC 277 Cognitive Interaction Technology.
+# The Excellence Cluster EXC 277 is a grant of the Deutsche
+# Forschungsgemeinschaft (DFG) in the context of the German
+# Excellence Initiative.
+
+from __future__ import division, print_function
+
+import os
+import platform
+import sys
+import time
+
+import traceback
+
+import ipaaca
+
+
+def exit(code):
+	ipaaca.exit(code)
+	#if platform.system() == 'Windows':
+	#	os._exit(code)
+	#else:
+	#	sys.exit(code)
+
+
+parser = ipaaca.IpaacaArgumentParser(description='Ipaaca Maintainor')
+
+
+if __name__ == '__main__':
+	arguments = parser.parse_args()
+
+	# Warn if user tries to configure a transport other than socket or a
+	# server mode other than server
+	if (ipaaca.defaults.IPAACA_DEFAULT_RSB_TRANSPORT is not None and
+	   		ipaaca.defaults.IPAACA_DEFAULT_RSB_TRANSPORT != 'socket'):
+		print("ERROR: Works only with RSB transport type 'socket'.")
+		ipaaca.exit(1)
+	if (ipaaca.defaults.IPAACA_DEFAULT_RSB_SOCKET_SERVER is not None and
+			ipaaca.defaults.IPAACA_DEFAULT_RSB_SOCKET_SERVER != '1'):
+		print("ERROR: Works only in socket server mode '0'.")
+		ipaaca.exit(1)
+
+	# Configure rsb socket transport
+	ipaaca.defaults.IPAACA_DEFAULT_RSB_TRANSPORT = 'socket'
+	ipaaca.defaults.IPAACA_DEFAULT_RSB_SOCKET_SERVER = '1'
+
+	ob = ipaaca.OutputBuffer("IPAACAMaintainor")
+
+	while True:
+		try:
+			time.sleep(1)
+		except KeyboardInterrupt:
+			decision = raw_input('Shutdown the IPAACA socket transport hub (y/[n])?')
+			if decision in 'yY':
+				break
+		except Exception, e:
+			print(u'Exception: '+unicode(traceback.format_exc()))
+			ipaaca.exit(1)
+	ipaaca.exit(0)