Worked with Sockets as part of SSAD Project

ads.pid

@@ -0,0 +1,3 @@
+LOGGER:5340
+ADAPTER:5365
+CONTROLLER:5390

config/adapters/centos_openvz_config.py

@@ -0,0 +1,11 @@
+# Settings for CentOSVZ Adapter
+TIMEOUT = 300
+supported_template = [
+    {'os': 'UBUNTU', 'version': '12.04',
+     'id': 'ubuntu-12.04-custom-with-emacs-x86_64'},
+    {'os': 'UBUNTU', 'version': '14.04',
+     'id': 'ubuntu-14.04_x86_64-custom-emacs-nginx'},
+    {'os': 'CENTOS', 'version': '12.04',
+     'id': 'centos-12.04-custum-x86_64'},
+    # {'os': 'DEBIAN', 'version': '7.0'}
+]

docs/provision.org

@@ -0,0 +1,205 @@
+#+TITLE: Provision ADS
+#+AUTHOR: VLEAD
+#+DATE: [2017-11-06 Mon]
+#+SETUPFILE: ./org-templates/level-0.org
+#+TAGS: boilerplate(b)
+#+EXCLUDE_TAGS: boilerplate
+#+OPTIONS: ^:nil
+
+* Introduction
+  This document describes how to provision and configure a
+  machine for ADS service.  Here provisioning implies
+  creation of machine and installation of all the necessary
+  software.  Configuration later configures settings to
+  enable the service. 
+
+
+* Machine
+  1. OS - CentOS 6.9
+  2. x86 64 bit architecture
+
+
+* Provisioning
+** Creation of machine
+   Machine with centos 6.9 operating system is assumed to be
+   available.  The machine is created in different ways on
+   different virtualizations.
+
+** Installation of Software
+   1. 
+
+
+* Settings for ADS 
+Execute the following steps, to configure and then run ADS as a service:
+
+** Edit the =ovpl/config/= directory
+   Copy =ovpl/config/sample_config.json= to =ovpl/config/config.json= and make
+   the following changes:
+
+  + To set the proxies.
+#+begin_src example
+    "ENVIRONMENT": {
+        "HTTP_PROXY":"http://proxy. vlabs.ac.in:8080",
+        "HTTPS_PROXY":"http://proxy.vlabs.ac.in:8080"
+    },
+#+end_src
+
+  + if no proxies are used, 
+#+begin_src example
+    "ENVIRONMENT": {
+        "HTTP_PROXY":"",
+        "HTTPS_PROXY":""
+    },
+#+end_src
+
+ + Set the COOKIE_SECRET value in CONTROLLER_CONFIG to some long randomly
+    generated string.
+
+#+begin_src example
+"COOKIE_SECRET": "<generate a long random UUID>",
+#+end_src  
+
+  + Set the SERVER_IP in LOGSERVER_CONFIGURATION to the IP address of the
+machine on which the ADS services are running.
+#+begin_src example
+"LOGSERVER_CONFIGURATION": {
+            "SERVER_IP": "ads-server-private-ip",
+	    },
+#+end_src
+
+  + Choose =POOLID= and =ADAPTERID= from =VPOOLS= section. Make the appropriate change in =ADAPTER_TO_USE=.
+    The =ADAPTERID= for =aws_adapter=, =centos_openvz_adapter= and =centos_bridged_adapter= is 1, 2 and 3 
+    respectively.
+
+#+begin_src example
+"ADAPTER_TO_USE" : {"POOLID" : 1, "ADAPTERID" : 2}
+#+end_src
+
+** Settings for =base_config=
+   1. Copy =config/adapters/sample_base_config.py= to =config/adapters/base_config.py=. 
+      For =aws_adapter=, no change in this file has to be done. For =centos_openvz_adapter=
+      and =centos_bridged_adapter= make the following changes in the file:
+
+      1. If the services are running on the base machine,
+         set =ADS_ON_CONTAINER= to False and copy public key (id_rsa.pub) to authorized_keys
+
+      2. If the services are running on a container, set =ADS_ON_CONTAINER= to
+         True and copy the public key of the container to the authorized_keys
+         of the base machine. 
+
+      3. Set BASE_IP_ADDRESS = "root@<IP>" where IP is the ip address of
+         base machine on which containers are created.
+
+      4. Set ADS_SERVER_VM_ID to CTID of container running ADS.
+         ADS_SERVER_VM_ID = "<CTID>" 
+
+      5. SUBNET field to match with the subnet of your base machine
+         If the ip address of your base machine is 10.2.58.XXX, 
+         SUBNET = ["10.2.58.12/28"]
+
+      6. =ADAPTER_NAME_SERVER=, set the field to a specific nameserver that is
+         used in the cluster or leave it as 'inherit' to use the nameserver
+         configuration from the base machine.
+
+      7. =SECRET_KEY=, used to authenticate the ADS WEB APP.
+
+** Settings for =centos_bridged_adapter=
+   Copy =config/adapters/sample_centos_bridged_config.py= to =config/adapters/centos_bridged_config.py=,
+   and change the =SUBNET_BRIDGE= in the file.
+
+** Settings for =centos_openvz_adapter=
+   - Copy =config/adapters/sample_centos_openvz_config.py= to
+     =config/adapters/centos_openvz_config.py=.
+** Settings for =aws_adapter=
+   Copy =config/adapters/sample_aws_config.py= to =config/adapters/aws_config.py=, and edit the values
+   accordingly. The private key =key_file_name.pem= should be copied inside =src/= directory.
+   See [[./docs/AWSAdapter.org][here]] for more details.
+
+** Commands to Manage the Service
+   + As root, go into =ovpl= directory and run =manage_services.sh= file:
+#+BEGIN_EXAMPLE
+$ cd ovpl
+$ ./manage_services (or) ./manage_services start
+#+END_EXAMPLE
+
+   + To stop the services
+#+BEGIN_EXAMPLE
+$ cd ovpl
+$ ./manage_services stop
+#+END_EXAMPLE
+
+  + To stop or start specific service use =./manage_services.sh [action]
+    [services]=
+    - [action]   = start (or) stop
+    - [services] = LOGGER (or) ADAPTER (or) CONTROLLER
+
+#+begin_example
+$ cd ovpl
+$ ./manage_services.sh start LOGGER
+#+end_example
+
++ For more help use =./manage_services.sh --help=.
+
+** Setting up ADS-web-app
+*** Install dependencies
+    This assumes apache server is installed. If not, please install the apache
+    server.
+#+BEGIN_EXAMPLE
+sudo su -
+yum update -y
+yum install epel-release
+yum install "mod_wsgi" -y
+yum install python-pip
+cd /root/ovpl/src/ads-web-app
+python setup.py install
+
+rsync -avz --progress /root/ovpl/src/ads-web-app /var/www/html/
+chmod -R 777 /var/www/html/ads-web-app
+#+END_EXAMPLE
+
+*** Setup Google Oauth credentials
+    - Create a project at [[https://console.developers.google.com][Google Dashboard]] and create =Oauth= credentails.
+    - Provide =CONSUMER_KEY= and =CONSUMER_SECRET= in
+      =/var/www/html/ads-web-app/config.py=.
+#+BEGIN_SRC 
+CONSUMER_KEY = "<consumerkey>"
+CONSUMER_SECRET = "<consumer secret key>"
+
+#+END_SRC
+*** Setup ADS credentials
+    - Provide =ADS= running server IP/Domain name to variable =ADS_URL=.
+    - Set secret key of =ADS= server to variable =ADS_SECRET_KEY=.
+#+BEGIN_SRC 
+ADS_URL = "http://<ip/domain>:5000"
+ADS_SECRET_KEY= "defaultadskey"
+
+#+END_SRC
+    - Add emails ids to =AUTHORIZED_USERS=
+#+BEGIN_SRC 
+AUTHORIZED_USERS = [
+    "[email protected]"
+]
+
+#+END_SRC
+
+*** Access =ADS= from browser
+   - Restart =Flask= server
+#+BEGIN_SRC 
+cd /var/www/html/ads-web-app/
+python app.py &
+#+END_SRC
+
+   - Open the location =http://ads-server-public-ip:8080= from the
+     browser and provide the lab id, lab sources url and version/tag.
+   - View the logs at /root/logs/ovpl.log by
+#+begin_src example
+tail -f /root/logs/ovpl.log
+#+end_src
+
+
+** Other related documentation:
+Steps to manually create a container
+-----
+1. vzctl create 101 --ostemplate ubuntu-12.04-custom-x86_64 --ipadd 10.2.58.3 --diskspace 10G:15.0G --hostname cse02.vlabs.ac.in
+2. vzctl start 101
+3. vzctl set 101 --nameserver inherit --ram 256M --swap 512M --onboot yes --save

src/ads-web-app/app.py

@@ -3,7 +3,12 @@
 from config import LOG_FILE_DIRECTORY
 from config import LOG_FILE
 from config import LOG_LEVEL
-import os
+from tornado import websocket, web, ioloop,httpserver,gen
+from tornado.wsgi import WSGIContainer
+from tornado.web import FallbackHandler, RequestHandler, Application
+import os, json,requests
+
+cl = []
 
 def create_app():
     # init our app
@@ -39,7 +44,37 @@ def configure_logging(app):
     app.logger.addHandler(log_handler)
     app.logger.setLevel(log_level)
 
+class SocketHandler(websocket.WebSocketHandler):
+    def check_origin(self, origin):
+        return True
+
+    def open(self):
+        if self not in cl:
+            cl.append(self)
+
+    def on_close(self):
+        if self in cl:
+            cl.remove(self)
+
+
+def send_msgs():
+    hello={
+        "value":"Ping"
+    }
+#    r = requests.get("http://localhost:7777/test")
+    [client.write_message(hello) for client in cl]
+
 #+NAME: run_server
 if __name__ == "__main__":
     app = create_app()
-    app.run(debug=True, host='0.0.0.0', threaded=True, port=8080)
+    tr = WSGIContainer(app)
+    run_app = Application(
+        handlers = [
+            (r"/ws", SocketHandler),
+            (r".*", FallbackHandler, dict(fallback=tr)),
+        ],
+        debug=True)
+    run = httpserver.HTTPServer(run_app)
+    run.listen(8080)
+    ioloop.PeriodicCallback(send_msgs, 1000).start()
+    ioloop.IOLoop.instance().start()

src/ads-web-app/config.py

@@ -1,10 +1,9 @@
 #Configure google id and secret key
-CONSUMER_KEY = "857206061842-11dem9m11firvs9164ig85q91vtnt8bd."\
-                "apps.googleusercontent.com"
-CONSUMER_SECRET = "WFEZLo8rTJ3uozckhIaNjQ5e"
+CONSUMER_KEY = "108110138830-k7p9bg01mm1j468g6rdmopaa13j1a6qf.apps.googleusercontent.com"
+CONSUMER_SECRET = "STbqt1Vlgs436omWj2Lv-unO"
 ADS_URL = "http://localhost:5000"
 ADS_SECRET_KEY= "defaultkey"
-APP_URL = "http://ads.virtual-labs.ac.in:8080"
+APP_URL = "http://local-ads.com:8080"
 # Configure your log paths
 LOG_FILE_DIRECTORY = 'logs'
 LOG_FILE = 'ads-web-app.log'
@@ -14,7 +13,5 @@
 LOG_LEVEL = 10
 
 AUTHORIZED_USERS = [
-    "[email protected]",
-    "[email protected]",
-    "[email protected]"
+    "[email protected]"
 ]

src/ads-web-app/logs/ads-web-app.log

@@ -0,0 +1,24 @@
+2017-08-17 17:32:22,656: DEBUG: api.py: index():53: lab_id = test, lab_src_url=https://github.com/vlead/lab-data-service.git, version=master, key=defaultkey, 
+2017-08-17 17:32:22,781: ERROR: api.py: index():88: error code = ('Connection aborted.', BadStatusLine('',))
+2017-08-17 17:32:36,838: DEBUG: api.py: index():53: lab_id = test, lab_src_url=https://github.com/vlead/lab-data-service.git, version=master, key=defaultkey, 
+2017-08-17 17:32:36,844: ERROR: api.py: index():88: error code = ('Connection aborted.', BadStatusLine('',))
+2017-08-17 17:36:44,212: DEBUG: api.py: index():53: lab_id = test, lab_src_url=https://github.com/vlead/lab-data-service.git, version=master, key=defaultkey, 
+2017-08-17 17:44:56,817: DEBUG: api.py: index():53: lab_id = ci, lab_src_url=https://github.com/vlead/continuous-integration.git, version=refactor, key=defaultkey, 
+2017-08-17 18:29:34,179: DEBUG: api.py: index():53: lab_id = ci, lab_src_url=https://github.com/vlead/continuous-integration.git, version=refactor, key=defaultkey, 
+2017-08-17 18:30:37,303: ERROR: api.py: index():88: error code = HTTPConnectionPool(host='localhost', port=5000): Max retries exceeded with url: / (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x1ac09d0>: Failed to establish a new connection: [Errno 110] Connection timed out',))
+2017-08-17 18:30:51,198: DEBUG: api.py: index():53: lab_id = ci, lab_src_url=https://github.com/vlead/continuous-integration.git, version=refactor, key=defaultkey, 
+2017-08-17 18:31:21,724: DEBUG: api.py: index():53: lab_id = ci, lab_src_url=https://github.com/vlead/continuous-integration.git, version=refactor, key=defaultkey, 
+2017-08-17 18:35:12,766: DEBUG: api.py: index():53: lab_id = ci, lab_src_url=https://github.com/vlead/continuous-integration.git, version=refactor, key=defaultkey, 
+2017-08-17 18:36:04,417: DEBUG: api.py: index():53: lab_id = ci, lab_src_url=https://github.com/vlead/continuous-integration.git, version=refactor, key=defaultkey, 
+2017-08-17 18:39:02,041: DEBUG: api.py: index():53: lab_id = ci, lab_src_url=https://github.com/vlead/continuous-integration.git, version=refactor, key=defaultkey, 
+2017-08-17 18:40:02,319: DEBUG: api.py: index():53: lab_id = ci, lab_src_url=https://github.com/vlead/continuous-integration.git, version=refactor, key=defaultkey, 
+2017-08-23 16:46:53,275: DEBUG: api.py: index():53: lab_id = test, lab_src_url=https://github.com/vlead/analytics-db, version=refactor, key=defaultkey, 
+2017-10-04 14:22:23,577: DEBUG: api.py: index():54: lab_id = test, lab_src_url=https://github.com/Virtual-Labs/computer-programming-responsive-iiith, version=master, key=defaultkey, 
+2017-10-04 14:23:48,182: DEBUG: api.py: index():54: lab_id = test, lab_src_url=https://github.com/Virtual-Labs/computer-programming-responsive-iiith, version=master, key=defaultkey, 
+2017-10-04 14:28:01,475: DEBUG: api.py: index():54: lab_id = test, lab_src_url=https://github.com/Virtual-Labs/computer-programming-responsive-iiith, version=master, key=defaultkey, 
+2017-11-13 07:52:31,392: DEBUG: api.py: index():54: lab_id = repear, lab_src_url=https://github.com/vlead/computer-programming-responsive, version=master, key=defaultkey, 
+2017-11-13 08:41:25,220: DEBUG: api.py: index():54: lab_id = test, lab_src_url=https://github.com/vlead/computer-programming-responsive, version=master, key=defaultkey, 
+2017-11-13 09:22:03,266: DEBUG: api.py: index():54: lab_id = test, lab_src_url=https://github.com/vlead/computer-programming-responsive, version=master, key=defaultkey, 
+2017-11-13 09:32:17,681: DEBUG: api.py: index():54: lab_id = test, lab_src_url=https://github.com/vlead/computer-programming-responsive, version=master, key=defaultkey, 
+2017-11-13 19:53:26,301: DEBUG: api.py: index():54: lab_id = i201, lab_src_url=0293, version=master, key=defaultkey, 
+2017-11-15 11:44:23,932: DEBUG: api.py: index():54: lab_id = test, lab_src_url=https://github.com/vlead/computer-programming-responsive, version=master, key=defaultkey, 

src/ads-web-app/requirem

@@ -0,0 +1,36 @@
+ADS==2.2
+argparse==1.2.1
+backports-abc==0.5
+backports.ssl-match-hostname==3.5.0.1
+boto==2.48.0
+bzr==2.1.1
+certifi==2017.7.27.1
+chardet==3.0.4
+click==6.7
+Flask==0.12.2
+Flask-OAuthlib==0.9.4
+idna==2.5
+iniparse==0.3.1
+itsdangerous==0.24
+Jinja2==2.9.6
+MarkupSafe==1.0
+mercurial==1.4
+netaddr==0.7.19
+oauthlib==2.0.2
+ordereddict==1.2
+OVPL==0.1
+paramiko==1.7.5
+pycrypto==2.0.1
+pycurl==7.19.0
+pygpgme==0.1
+pymongo==3.4.0
+requests==2.18.2
+requests-oauthlib==0.8.0
+sh==1.12.14
+singledispatch==3.4.0.3
+six==1.10.0
+tornado==4.5.1
+urlgrabber==3.9.1
+urllib3==1.22
+Werkzeug==0.12.2
+yum-metadata-parser==1.1.2

src/ads-web-app/templates/base.html

@@ -9,6 +9,7 @@
     <!-- Bootstrap -->
     <link rel="stylesheet" href="//netdna.bootstrapcdn.com/bootstrap/3.1.0/css/bootstrap.min.css">
     <link rel="stylesheet" href="static/css/main.css">
+    <script type="text/javascript" src="//ajax.googleapis.com/ajax/libs/jquery/1.8.2/jquery.min.js"></script>
   </head>
   <body>
     <div class="navbar navbar-default navbar-custom">