Don't save job filtered job lists in variables. Handle service not found error.

app/scheduler_functions.py

@@ -10,11 +10,7 @@ def checkout_jobs():
     with scheduler.app.app_context():
         client = docker.from_env()
         jobs = db.session.query(Job)
-        submitted_jobs = jobs.filter_by(status='submitted').all()
-        foo_jobs = jobs.filter(Job.status != 'complete',
-                               Job.status != 'failed',
-                               Job.status != 'submitted').all()
-        for job in submitted_jobs:
+        for job in jobs.filter_by(status='submitted').all():
             _command = (job.service
                         + ' -i /files'
                         + ' -o /files/output'
@@ -65,13 +61,15 @@ def checkout_jobs():
                 restart_policy=_restart_policy
             )
             job.status = 'scheduled'
-        for job in foo_jobs:
-            '''
-            ' TODO: Handle service not found error.
-            '''
-            service = client.services.get(str(job.id))
-            job.status = service.tasks()[0].get('Status').get('State')
-            if job.status == 'complete' or job.status == 'failed':
-                job.end_date = datetime.utcnow()
-                service.remove()
+        for job in jobs.filter(Job.status != 'complete',
+                               Job.status != 'failed',
+                               Job.status != 'submitted').all():
+            try:
+                service = client.services.get(str(job.id))
+                job.status = service.tasks()[0].get('Status').get('State')
+                if job.status == 'complete' or job.status == 'failed':
+                    job.end_date = datetime.utcnow()
+                    service.remove()
+            except docker.errors.NotFound:
+                job.status = 'failed'
         db.session.commit()