Merge local master to dev

.gitlab-ci.yml

 image: python:latest
 
 stages:
-  - pretest
   - build
   - test
   - coverage
   - doc_build
 
-ssh:
-  stage: pretest
-  before_script:
-  - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
-  - 'which rsync || apt-get install rsync -y'
-  - eval $(ssh-agent -s)
-  - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
-  - mkdir -p ~/.ssh
-  - chmod 700 ~/.ssh
-  - whoami
-  - hostname
-  - echo "$SSH_PRIVATE_KEY" > ~/.ssh/gitlab_rsa
-  - echo "$SSH_PUBLIC_KEY" > ~/.ssh/gitlab_rsa.pub
-  
-    
-  script:
-  - ssh -p50022 bolirev@bioneuro77.biologie.uni-bielefeld.de "mkdir -p /fastdata/html/navipy/"
-
 navipy_install:
   stage: build
   before_script:
@@ -40,7 +21,6 @@ navipy_install:
   artifacts:
     paths:
       - venv
-      - build
     expire_in: 2 hours
 
   artifacts:

README.md

@@ -11,22 +11,67 @@ The navigation toolbox aims to bring in an intuitive python toolbox different me
 -    Avoid re-rendering by using grid constrained motion.
 
 # How to install the navigation toolbox
+The rendering from the insect point of view is done with the blender rendering engine. Thus, you will need to first install Blender
+https://www.blender.org/
 
+## Windows
+We recommend using Anaconda (https://www.anaconda.com/) and create a virtual environment within it before installing the toolbox. 
+
+Start the Anaconda Prompt, and then enter
+```
+conda update conda
+```
+Upadate any packages if necessary by typing y to proceed
+
+Then, create a virtual environment for your project
+```
+conda create -n yourenvname python=x.x anaconda
+```
+here `yourenvname` is the name of your project (without special characters and spaces), and `python=x.x` the version of python
+to use, for example `python=3.5.3` (see table below to install a version matching with blender)
+
+You can now activate your environment. 
+```
+activate yourenvname
+```
+
+and install navipy by going the root folder of the toolbox (use cd to change directory and dir to list the content of a directory), and typing
 ```
 python setup.py install
 ```
 
-# Code of conduct
+You can now use the navigation toolbox. 
 
-In the interest of fostering an open and welcoming environment, we as students and teachers pledge to making participation in this class a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
+## Linux (Ubuntu)
+From the terminal 
+```
+pip install update
+```
+Upadate any packages if necessary by typing y to proceed
 
-# Todo:
+Then, create a virtual environment for your project
+```
+mkvirtualenv yourenvname
+```
+here `yourenvname` is the name of your project (without special characters and spaces)
+
+You can now activate your environment. 
+```
+workon yourenvname
+```
+
+and install navipy by going the root folder of the toolbox (use cd to change directory and ls to list the content of a directory), and typing
+```
+python setup.py install
+```
 
-1. solve problems with CI/CD pipelines for contineous integration
-2. add tutorials
-3. comment test function -> add to doc
+## Blender-python version
+| Blender version | Python version |
+| --------------- | -------------- |
+| 2.79b           | 3.5.3          |
 
-# Useful links
 
-## for testing gitlab-ci locally
-https://substrakt.com/how-to-debug-gitlab-ci-builds-locally/
+
+# Code of conduct
+
+In the interest of fostering an open and welcoming environment, we as users and developers pledge to making participation with this project a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
\ No newline at end of file

doc/source/tutorials/02a-orientation-background.ipynb

@@ -525,7 +525,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.5"
+   "version": "3.6.3"
   }
  },
  "nbformat": 4,

navipy/resources/sample_experiment/Lobecke_JEB_2018/fullcalib_fromtra.xml

+<?xml version="1.0"?>
+<opencv_storage>
+<ncameras>2.</ncameras>
+ <!-- resumed -->
+
+<pose_0 type_id="opencv-matrix">
+  <rows>4</rows>
+  <cols>4</cols>
+  <dt>d</dt>
+  <data>
+    -9.9975884141929716e-01 -1.9663083577624682e-02
+    -9.7786577894851904e-03 2.3960770101324730e+01
+    1.2596475289801174e-02 -8.7822032197929456e-01
+    4.7809036266469301e-01 -1.7450255985791409e+02
+    -1.7988546751139770e-02 4.7785189045017246e-01
+    8.7825621715930891e-01 -1.2689224150808659e+03 0. 0. 0. 1.</data></pose_0>
+ <!-- resumed -->
+
+<intrinsic_matrix_0 type_id="opencv-matrix">
+  <rows>3</rows>
+  <cols>3</cols>
+  <dt>d</dt>
+  <data>
+    1.3854069839766423e+03 0. 1.0142369229097144e+03 0.
+    1.3860018990405542e+03 9.6972229918852702e+02 0. 0. 1.</data></intrinsic_matrix_0>
+ <!-- resumed -->
+
+<distortion_0 type_id="opencv-matrix">
+  <rows>5</rows>
+  <cols>1</cols>
+  <dt>d</dt>
+  <data>
+    -1.6496133317304024e-01 7.6541851811227621e-02
+    1.4703239618294489e-04 -5.0753504623544379e-04
+    5.9159458153562314e-02</data></distortion_0>
+ <!-- resumed -->
+
+<pose_1 type_id="opencv-matrix">
+  <rows>4</rows>
+  <cols>4</cols>
+  <dt>d</dt>
+  <data>
+    -9.9999925934757616e-01 4.6011102855678676e-05
+    -1.2162184333542050e-03 9.1657057770283288e+00
+    -4.6920052752197648e-05 -9.9999971964107015e-01
+    7.4733947436152800e-04 4.1419559776785363e+01
+    -1.2161837064630836e-03 7.4739598587578605e-04
+    9.9999898114769714e-01 -1.1865049751543168e+03 0. 0. 0. 1.</data></pose_1>
+ <!-- resumed -->
+
+<intrinsic_matrix_1 type_id="opencv-matrix">
+  <rows>3</rows>
+  <cols>3</cols>
+  <dt>d</dt>
+  <data>
+    1.3877924464919429e+03 0. 1.0250625063928485e+03 0.
+    1.3892469602478291e+03 1.0633879325646449e+03 0. 0. 1.</data></intrinsic_matrix_1>
+ <!-- resumed -->
+
+<distortion_1 type_id="opencv-matrix">
+  <rows>5</rows>
+  <cols>1</cols>
+  <dt>d</dt>
+  <data>
+    -1.6987468870812769e-01 1.0743242290093688e-01
+    1.9614281508288295e-04 -6.6638746010521906e-04
+    -8.9751502378932948e-03</data></distortion_1>
+</opencv_storage>

navipy/tools/plots.py

@@ -35,7 +35,7 @@ corresponding color-map (scalar mappable)
     """
 
     frame_series = pd.Series(index=np.arange(
-        min(frame_range), max(frame_range)+1))  # +1 include the last frame
+        min(frame_range), max(frame_range) + 1))  # +1 include the last frame
     frame_series[:] = np.linspace(0, 1, frame_series.shape[0])
     return get_color_dataframe(frame_series, cmap=cmap)
 
@@ -70,9 +70,10 @@ index as in series.
     # substract offset from values, so the smalles one is zero
     normalised_values -= normalised_values.min()
     # all values are zero and have the same value
-    assert normalised_values.max() != 0, 'series.values are constant'
+    if normalised_values.max() == 0:
+        raise ValueError('series.values are constant')
     # actually normalize the values
-    normalised_values /= normalised_values.max()
+    normalised_values = normalised_values / normalised_values.max()
     colors = cmap(normalised_values)
     # create the dataframe from color
     df_colors = pd.DataFrame(
@@ -117,9 +118,9 @@ def draw_frame(frame,
     origin = frame[:, 3]
     for (i, color) in enumerate(colors):
         v = frame[:, i]
-        xs = [origin[0], origin[0]+v[0]]
-        ys = [origin[1], origin[1]+v[1]]
-        zs = [origin[2], origin[2]+v[2]]
+        xs = [origin[0], origin[0] + v[0]]
+        ys = [origin[1], origin[1] + v[1]]
+        zs = [origin[2], origin[2] + v[2]]
         a = Arrow3D(xs, ys, zs,
                     mutation_scale=mutation_scale,
                     lw=lw,

navipy/trajectories/__init__.py

@@ -10,6 +10,11 @@ import navipy.maths.homogeneous_transformations as htf
 from navipy.errorprop import propagate_error
 from .transformations import markers2translate, markers2euler
 from navipy.tools.plots import get_color_dataframe
+import matplotlib.pyplot as plt
+from mpl_toolkits.mplot3d import Axes3D  # noqa F401
+from multiprocessing import Pool
+from functools import partial
+import time
 
 
 def _markers2position(x, kwargs):
@@ -32,6 +37,18 @@ def _markers2angles(x, kwargs):
                          triangle_mode, euler_axes)
 
 
+def _markerstransform(index_i, trajectory,
+                      homogeneous_markers, rotation_mode):
+    row = trajectory.loc[index_i]
+    angles = row.loc[rotation_mode].values
+    translate = row.loc['location'].values
+    trans_mat = htf.compose_matrix(angles=angles,
+                                   translate=translate,
+                                   axes=rotation_mode)
+    tmarker = trans_mat.dot(homogeneous_markers)[:3]
+    return tmarker.transpose().flatten()
+
+
 class Trajectory(pd.DataFrame):
     def __init__(self, rotconv='rzyx', indeces=np.arange(1)):
         columns = self.__build_columns(rotconv)
@@ -446,7 +463,7 @@ class Trajectory(pd.DataFrame):
             self.loc[index_i, self.rotation_mode] = orientation
         return self
 
-    def world2body(self, markers):
+    def world2body(self, markers, indeces=None):
         """ Transform markers in world coordinate to body coordinate
         """
         if not isinstance(markers, pd.Series):
@@ -458,28 +475,39 @@ class Trajectory(pd.DataFrame):
             msg += ' (i,"x"), (i,"y"),(i,"z")\n'
             msg += 'here i is the index of the marker'
             raise TypeError(msg)
-        transformed_markers = pd.DataFrame(index=self.index,
-                                           columns=markers.index,
-                                           dtype=float)
-        # Looping through each time point along the trajectory
+        if indeces is None:
+            # Looping through each time point along the trajectory
+            indeces = self.index
+        # More than one marker may be transformed
+        # The marker are assume to be a multiIndex dataframe
+        homogeneous_markers = markers.unstack()
+        homogeneous_markers['w'] = 1
+        homogeneous_markers = homogeneous_markers.transpose()
+        # Looping throught the indeces
         # to get the homogeneous transformation from the position orientation
         # and then apply the transformed to the marker position
-        for index_i, row in self.iterrows():
+        with Pool() as p:
+            result = p.map(
+                partial(_markerstransform,
+                        trajectory=self,
+                        homogeneous_markers=homogeneous_markers,
+                        rotation_mode=self.rotation_mode),
+                indeces)
+        transformed_markers = pd.DataFrame(data=result,
+                                           index=indeces,
+                                           columns=markers.index,
+                                           dtype=float)
+        return transformed_markers
+
+        for index_i in indeces:
+            row = self.loc[index_i]
             angles = row.loc[self.rotation_mode].values
             translate = row.loc['location'].values
             trans_mat = htf.compose_matrix(angles=angles,
                                            translate=translate,
                                            axes=self.rotation_mode)
-            # More than one marker may be transformed
-            # The marker are assume to be a multiIndex dataframe
-            for marki in markers.index.levels[0]:
-                markpos = [markers.loc[marki, 'x'],
-                           markers.loc[marki, 'y'],
-                           markers.loc[marki, 'z'],
-                           1]
-                markpos = trans_mat.dot(markpos)[:3]
-                transformed_markers.loc[index_i,
-                                        (marki, ['x', 'y', 'z'])] = markpos
+            tmarker = trans_mat.dot(homogeneous_markers.transpose())[:3]
+            transformed_markers.loc[index_i, :] = tmarker.transpose().flatten()
 
         return transformed_markers
 
@@ -515,11 +543,11 @@ class Trajectory(pd.DataFrame):
                                    'dalpha_2']] = rot.squeeze()
         return velocity
 
-    def lollipops(self, ax,
+    def lollipops(self, ax=None,
                   colors=None, step_lollipop=1,
                   offset_lollipop=0, lollipop_marker='o',
-                  linewidth=1, lollipop_tail_width=1,
-                  lollipop_head_size=1
+                  linewidth=1, lollipop_tail_width=1, lollipop_tail_length=1,
+                  lollipop_head_size=1, stickdir='backward'
                   ):
         """ lollipops plot
 
@@ -543,26 +571,38 @@ class Trajectory(pd.DataFrame):
         :param step_lollipop: number of frames between two lollipops
         :param offset_lollipop: the first lollipop to be plotted
         :param lollipop_marker: the head of the lollipop
-        :param linewidth:
-        :param lollipop_tail_width:
-        :param lollipop_head_size:
+        :param linewidth: The width of the line connecting lollipops
+        :param lollipop_tail_width: The width of the lollipop stick
+        :param lollipop_tail_length: The length of the lollipop stick
+        :param lollipop_head_size: The size of the lollipop
+        :param stickdir: The direction of the stick of the animal \
+(backward or forward)
         """
+        # import time
+        t_start = time.time()
+        if ax is None:
+            fig = plt.figure()
+            ax = fig.add_subplot(111, projection='3d')
         direction = self.facing_direction()
         if colors is None:
             timeseries = pd.Series(data=self.index,
                                    index=self.index)
             colors, sm = get_color_dataframe(timeseries)
-        # Create a continuous index from colors
-        frames = np.arange(colors.index.min(), colors.index.max() + 1)
-
+        # Create a continuous index from trajectory
+        frames = np.arange(self.index.min(), self.index.max() + 1)
+        # Select indeces to save time
+        indeces = frames[offset_lollipop::step_lollipop]
         # Calculate agent tail direction
         # use function in trajectory to get any point bodyref to worldref
         tailmarker = pd.Series(data=0,
                                index=pd.MultiIndex.from_product(
                                    [[0],
                                     ['x', 'y', 'z']]))
-        tailmarker.loc[0, 'x'] = 1
-        tail = self.world2body(tailmarker)
+        if stickdir == 'forward':
+            tailmarker.loc[0, 'x'] = -lollipop_tail_length
+        else:
+            tailmarker.loc[0, 'x'] = lollipop_tail_length
+        tail = self.world2body(tailmarker, indeces=indeces)
         tail = tail.loc[:, 0]
         # Plot the agent trajectory
         # - loop through consecutive point
@@ -571,36 +611,49 @@ class Trajectory(pd.DataFrame):
         x = self.loc[:, ('location', 'x')]
         y = self.loc[:, ('location', 'y')]
         z = self.loc[:, ('location', 'z')]
-
-        for frame_i, frame_j in zip(frames[:-1], frames[1:]):
-            # Frames may be missing in trajectory,
-            # and therefore can not be plotted
-            if (frame_i in self.index) and \
-                    (frame_j in self.index) and \
-                    (frame_i in self.index):
-                color = [colors.r[frame_i], colors.g[frame_i],
-                         colors.b[frame_i], colors.a[frame_i]]
-                # Create the line to plot
-                line_x = [x[frame_i], x[frame_j]]
-                line_y = [y[frame_i], y[frame_j]]
-                line_z = [z[frame_i], z[frame_j]]
-                # Actual plot command
-                ax.plot(xs=line_x, ys=line_y, zs=line_z,
-                        color=color, linewidth=linewidth)
+        print(time.time() - t_start)
+        t_start = time.time()
+        if isinstance(colors, pd.DataFrame):
+            # Each segment will be plotted with a different color
+            # we therefore need to loop through all points
+            # in the trajectory, a rather long process
+            for frame_i, frame_j in zip(frames[:-1], frames[1:]):
+                # Frames may be missing in trajectory,
+                # and therefore can not be plotted
+                if (frame_i in self.index) and \
+                   (frame_j in self.index) and \
+                   (frame_i in self.index):
+                    color = [colors.r[frame_i], colors.g[frame_i],
+                             colors.b[frame_i], colors.a[frame_i]]
+                    # Create the line to plot
+                    line_x = [x[frame_i], x[frame_j]]
+                    line_y = [y[frame_i], y[frame_j]]
+                    line_z = [z[frame_i], z[frame_j]]
+                    # Actual plot command
+                    ax.plot(xs=line_x, ys=line_y, zs=line_z,
+                            color=color, linewidth=linewidth)
+        else:
+            ax.plot(xs=x, ys=y, zs=z, color=colors, linewidth=linewidth)
+        print(time.time() - t_start)
+        t_start = time.time()
         # Plot the lollipop
         # - loop through the frames with a step of step_lollipop
         # - The lollipop is colored with the color of this frame
         # - Each lollipop is composed of a marker,
         # a point on the agent trajectory
         #   and a line representing the body (anti facing direction)
-        for frame_i in frames[offset_lollipop::step_lollipop]:
+        for frame_i in indeces:
             # Frames may be missing in trajectory,
             # and therefore can not be plotted
             if (frame_i in self.index) and  \
-                    (frame_i in direction.index) and \
-                    (frame_i in colors.index):
-                color = [colors.r[frame_i], colors.g[frame_i],
-                         colors.b[frame_i], colors.a[frame_i]]
+                    (frame_i in direction.index):
+                if isinstance(colors, pd.DataFrame):
+                    if frame_i not in colors.index:
+                        continue
+                    color = [colors.r[frame_i], colors.g[frame_i],
+                             colors.b[frame_i], colors.a[frame_i]]
+                else:
+                    color = colors
                 # Create the line to plot
                 line_x = [self.x[frame_i],
                           tail.x[frame_i]]
@@ -617,3 +670,4 @@ class Trajectory(pd.DataFrame):
                         color=color,
                         marker=lollipop_marker,
                         markersize=lollipop_head_size)
+        print(time.time() - t_start)