From 375c0fda8d7f8c8dac2d40881b659bd83ab18a3e Mon Sep 17 00:00:00 2001 From: root Date: Wed, 15 Nov 2017 13:08:52 +0000 Subject: [PATCH] Added socket channel --- ads.pid | 3 + config/adapters/centos_openvz_config.py | 11 ++ docs/provision.org | 205 ++++++++++++++++++++++++ src/ads-web-app/app.py | 39 ++++- src/ads-web-app/config.py | 11 +- src/ads-web-app/logs/ads-web-app.log | 24 +++ src/ads-web-app/requirem | 36 +++++ src/ads-web-app/requirement.txt | 0 src/ads-web-app/templates/base.html | 1 + src/ads-web-app/templates/index.html | 29 ++++ src/httplogging/helper.py | 15 ++ src/httplogging/http_logging_server.py | 10 ++ 12 files changed, 375 insertions(+), 9 deletions(-) create mode 100644 ads.pid create mode 100644 config/adapters/centos_openvz_config.py create mode 100644 docs/provision.org create mode 100755 src/ads-web-app/logs/ads-web-app.log create mode 100644 src/ads-web-app/requirem create mode 100644 src/ads-web-app/requirement.txt diff --git a/ads.pid b/ads.pid new file mode 100644 index 0000000..a3323f9 --- /dev/null +++ b/ads.pid @@ -0,0 +1,3 @@ +LOGGER:5340 +ADAPTER:5365 +CONTROLLER:5390 diff --git a/config/adapters/centos_openvz_config.py b/config/adapters/centos_openvz_config.py new file mode 100644 index 0000000..00e4b34 --- /dev/null +++ b/config/adapters/centos_openvz_config.py @@ -0,0 +1,11 @@ +# Settings for CentOSVZ Adapter +TIMEOUT = 300 +supported_template = [ + {'os': 'UBUNTU', 'version': '12.04', + 'id': 'ubuntu-12.04-custom-with-emacs-x86_64'}, + {'os': 'UBUNTU', 'version': '14.04', + 'id': 'ubuntu-14.04_x86_64-custom-emacs-nginx'}, + {'os': 'CENTOS', 'version': '12.04', + 'id': 'centos-12.04-custum-x86_64'}, + # {'os': 'DEBIAN', 'version': '7.0'} +] diff --git a/docs/provision.org b/docs/provision.org new file mode 100644 index 0000000..ae8a046 --- /dev/null +++ b/docs/provision.org @@ -0,0 +1,205 @@ +#+TITLE: Provision ADS +#+AUTHOR: VLEAD +#+DATE: [2017-11-06 Mon] +#+SETUPFILE: ./org-templates/level-0.org +#+TAGS: boilerplate(b) +#+EXCLUDE_TAGS: boilerplate +#+OPTIONS: ^:nil + +* Introduction + This document describes how to provision and configure a + machine for ADS service. Here provisioning implies + creation of machine and installation of all the necessary + software. Configuration later configures settings to + enable the service. + + +* Machine + 1. OS - CentOS 6.9 + 2. x86 64 bit architecture + + +* Provisioning +** Creation of machine + Machine with centos 6.9 operating system is assumed to be + available. The machine is created in different ways on + different virtualizations. + +** Installation of Software + 1. + + +* Settings for ADS +Execute the following steps, to configure and then run ADS as a service: + +** Edit the =ovpl/config/= directory + Copy =ovpl/config/sample_config.json= to =ovpl/config/config.json= and make + the following changes: + + + To set the proxies. +#+begin_src example + "ENVIRONMENT": { + "HTTP_PROXY":"http://proxy. vlabs.ac.in:8080", + "HTTPS_PROXY":"http://proxy.vlabs.ac.in:8080" + }, +#+end_src + + + if no proxies are used, +#+begin_src example + "ENVIRONMENT": { + "HTTP_PROXY":"", + "HTTPS_PROXY":"" + }, +#+end_src + + + Set the COOKIE_SECRET value in CONTROLLER_CONFIG to some long randomly + generated string. + +#+begin_src example +"COOKIE_SECRET": "", +#+end_src + + + Set the SERVER_IP in LOGSERVER_CONFIGURATION to the IP address of the +machine on which the ADS services are running. +#+begin_src example +"LOGSERVER_CONFIGURATION": { + "SERVER_IP": "ads-server-private-ip", + }, +#+end_src + + + Choose =POOLID= and =ADAPTERID= from =VPOOLS= section. Make the appropriate change in =ADAPTER_TO_USE=. + The =ADAPTERID= for =aws_adapter=, =centos_openvz_adapter= and =centos_bridged_adapter= is 1, 2 and 3 + respectively. + +#+begin_src example +"ADAPTER_TO_USE" : {"POOLID" : 1, "ADAPTERID" : 2} +#+end_src + +** Settings for =base_config= + 1. Copy =config/adapters/sample_base_config.py= to =config/adapters/base_config.py=. + For =aws_adapter=, no change in this file has to be done. For =centos_openvz_adapter= + and =centos_bridged_adapter= make the following changes in the file: + + 1. If the services are running on the base machine, + set =ADS_ON_CONTAINER= to False and copy public key (id_rsa.pub) to authorized_keys + + 2. If the services are running on a container, set =ADS_ON_CONTAINER= to + True and copy the public key of the container to the authorized_keys + of the base machine. + + 3. Set BASE_IP_ADDRESS = "root@" where IP is the ip address of + base machine on which containers are created. + + 4. Set ADS_SERVER_VM_ID to CTID of container running ADS. + ADS_SERVER_VM_ID = "" + + 5. SUBNET field to match with the subnet of your base machine + If the ip address of your base machine is 10.2.58.XXX, + SUBNET = ["10.2.58.12/28"] + + 6. =ADAPTER_NAME_SERVER=, set the field to a specific nameserver that is + used in the cluster or leave it as 'inherit' to use the nameserver + configuration from the base machine. + + 7. =SECRET_KEY=, used to authenticate the ADS WEB APP. + +** Settings for =centos_bridged_adapter= + Copy =config/adapters/sample_centos_bridged_config.py= to =config/adapters/centos_bridged_config.py=, + and change the =SUBNET_BRIDGE= in the file. + +** Settings for =centos_openvz_adapter= + - Copy =config/adapters/sample_centos_openvz_config.py= to + =config/adapters/centos_openvz_config.py=. +** Settings for =aws_adapter= + Copy =config/adapters/sample_aws_config.py= to =config/adapters/aws_config.py=, and edit the values + accordingly. The private key =key_file_name.pem= should be copied inside =src/= directory. + See [[./docs/AWSAdapter.org][here]] for more details. + +** Commands to Manage the Service + + As root, go into =ovpl= directory and run =manage_services.sh= file: +#+BEGIN_EXAMPLE +$ cd ovpl +$ ./manage_services (or) ./manage_services start +#+END_EXAMPLE + + + To stop the services +#+BEGIN_EXAMPLE +$ cd ovpl +$ ./manage_services stop +#+END_EXAMPLE + + + To stop or start specific service use =./manage_services.sh [action] + [services]= + - [action] = start (or) stop + - [services] = LOGGER (or) ADAPTER (or) CONTROLLER + +#+begin_example +$ cd ovpl +$ ./manage_services.sh start LOGGER +#+end_example + ++ For more help use =./manage_services.sh --help=. + +** Setting up ADS-web-app +*** Install dependencies + This assumes apache server is installed. If not, please install the apache + server. +#+BEGIN_EXAMPLE +sudo su - +yum update -y +yum install epel-release +yum install "mod_wsgi" -y +yum install python-pip +cd /root/ovpl/src/ads-web-app +python setup.py install + +rsync -avz --progress /root/ovpl/src/ads-web-app /var/www/html/ +chmod -R 777 /var/www/html/ads-web-app +#+END_EXAMPLE + +*** Setup Google Oauth credentials + - Create a project at [[https://console.developers.google.com][Google Dashboard]] and create =Oauth= credentails. + - Provide =CONSUMER_KEY= and =CONSUMER_SECRET= in + =/var/www/html/ads-web-app/config.py=. +#+BEGIN_SRC +CONSUMER_KEY = "" +CONSUMER_SECRET = "" + +#+END_SRC +*** Setup ADS credentials + - Provide =ADS= running server IP/Domain name to variable =ADS_URL=. + - Set secret key of =ADS= server to variable =ADS_SECRET_KEY=. +#+BEGIN_SRC +ADS_URL = "http://:5000" +ADS_SECRET_KEY= "defaultadskey" + +#+END_SRC + - Add emails ids to =AUTHORIZED_USERS= +#+BEGIN_SRC +AUTHORIZED_USERS = [ + "heregoesemail@gmail.com" +] + +#+END_SRC + +*** Access =ADS= from browser + - Restart =Flask= server +#+BEGIN_SRC +cd /var/www/html/ads-web-app/ +python app.py & +#+END_SRC + + - Open the location =http://ads-server-public-ip:8080= from the + browser and provide the lab id, lab sources url and version/tag. + - View the logs at /root/logs/ovpl.log by +#+begin_src example +tail -f /root/logs/ovpl.log +#+end_src + + +** Other related documentation: +Steps to manually create a container +----- +1. vzctl create 101 --ostemplate ubuntu-12.04-custom-x86_64 --ipadd 10.2.58.3 --diskspace 10G:15.0G --hostname cse02.vlabs.ac.in +2. vzctl start 101 +3. vzctl set 101 --nameserver inherit --ram 256M --swap 512M --onboot yes --save diff --git a/src/ads-web-app/app.py b/src/ads-web-app/app.py index 6df290e..6e9051b 100644 --- a/src/ads-web-app/app.py +++ b/src/ads-web-app/app.py @@ -3,7 +3,12 @@ from config import LOG_FILE_DIRECTORY from config import LOG_FILE from config import LOG_LEVEL -import os +from tornado import websocket, web, ioloop,httpserver,gen +from tornado.wsgi import WSGIContainer +from tornado.web import FallbackHandler, RequestHandler, Application +import os, json,requests + +cl = [] def create_app(): # init our app @@ -39,7 +44,37 @@ def configure_logging(app): app.logger.addHandler(log_handler) app.logger.setLevel(log_level) +class SocketHandler(websocket.WebSocketHandler): + def check_origin(self, origin): + return True + + def open(self): + if self not in cl: + cl.append(self) + + def on_close(self): + if self in cl: + cl.remove(self) + + +def send_msgs(): + hello={ + "value":"Ping" + } +# r = requests.get("http://localhost:7777/test") + [client.write_message(hello) for client in cl] + #+NAME: run_server if __name__ == "__main__": app = create_app() - app.run(debug=True, host='0.0.0.0', threaded=True, port=8080) + tr = WSGIContainer(app) + run_app = Application( + handlers = [ + (r"/ws", SocketHandler), + (r".*", FallbackHandler, dict(fallback=tr)), + ], + debug=True) + run = httpserver.HTTPServer(run_app) + run.listen(8080) + ioloop.PeriodicCallback(send_msgs, 1000).start() + ioloop.IOLoop.instance().start() diff --git a/src/ads-web-app/config.py b/src/ads-web-app/config.py index 945a2f0..58d0eae 100644 --- a/src/ads-web-app/config.py +++ b/src/ads-web-app/config.py @@ -1,10 +1,9 @@ #Configure google id and secret key -CONSUMER_KEY = "857206061842-11dem9m11firvs9164ig85q91vtnt8bd."\ - "apps.googleusercontent.com" -CONSUMER_SECRET = "WFEZLo8rTJ3uozckhIaNjQ5e" +CONSUMER_KEY = "108110138830-k7p9bg01mm1j468g6rdmopaa13j1a6qf.apps.googleusercontent.com" +CONSUMER_SECRET = "STbqt1Vlgs436omWj2Lv-unO" ADS_URL = "http://localhost:5000" ADS_SECRET_KEY= "defaultkey" -APP_URL = "http://ads.virtual-labs.ac.in:8080" +APP_URL = "http://local-ads.com:8080" # Configure your log paths LOG_FILE_DIRECTORY = 'logs' LOG_FILE = 'ads-web-app.log' @@ -14,7 +13,5 @@ LOG_LEVEL = 10 AUTHORIZED_USERS = [ - "kammari.sripathi@gmail.com", - "madhavipuliraju@gmail.com", - "heregoesnextemail@gmail.com" + "16101998ms@gmail.com" ] diff --git a/src/ads-web-app/logs/ads-web-app.log b/src/ads-web-app/logs/ads-web-app.log new file mode 100755 index 0000000..4d96915 --- /dev/null +++ b/src/ads-web-app/logs/ads-web-app.log @@ -0,0 +1,24 @@ +2017-08-17 17:32:22,656: DEBUG: api.py: index():53: lab_id = test, lab_src_url=https://github.com/vlead/lab-data-service.git, version=master, key=defaultkey, +2017-08-17 17:32:22,781: ERROR: api.py: index():88: error code = ('Connection aborted.', BadStatusLine('',)) +2017-08-17 17:32:36,838: DEBUG: api.py: index():53: lab_id = test, lab_src_url=https://github.com/vlead/lab-data-service.git, version=master, key=defaultkey, +2017-08-17 17:32:36,844: ERROR: api.py: index():88: error code = ('Connection aborted.', BadStatusLine('',)) +2017-08-17 17:36:44,212: DEBUG: api.py: index():53: lab_id = test, lab_src_url=https://github.com/vlead/lab-data-service.git, version=master, key=defaultkey, +2017-08-17 17:44:56,817: DEBUG: api.py: index():53: lab_id = ci, lab_src_url=https://github.com/vlead/continuous-integration.git, version=refactor, key=defaultkey, +2017-08-17 18:29:34,179: DEBUG: api.py: index():53: lab_id = ci, lab_src_url=https://github.com/vlead/continuous-integration.git, version=refactor, key=defaultkey, +2017-08-17 18:30:37,303: ERROR: api.py: index():88: error code = HTTPConnectionPool(host='localhost', port=5000): Max retries exceeded with url: / (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 110] Connection timed out',)) +2017-08-17 18:30:51,198: DEBUG: api.py: index():53: lab_id = ci, lab_src_url=https://github.com/vlead/continuous-integration.git, version=refactor, key=defaultkey, +2017-08-17 18:31:21,724: DEBUG: api.py: index():53: lab_id = ci, lab_src_url=https://github.com/vlead/continuous-integration.git, version=refactor, key=defaultkey, +2017-08-17 18:35:12,766: DEBUG: api.py: index():53: lab_id = ci, lab_src_url=https://github.com/vlead/continuous-integration.git, version=refactor, key=defaultkey, +2017-08-17 18:36:04,417: DEBUG: api.py: index():53: lab_id = ci, lab_src_url=https://github.com/vlead/continuous-integration.git, version=refactor, key=defaultkey, +2017-08-17 18:39:02,041: DEBUG: api.py: index():53: lab_id = ci, lab_src_url=https://github.com/vlead/continuous-integration.git, version=refactor, key=defaultkey, +2017-08-17 18:40:02,319: DEBUG: api.py: index():53: lab_id = ci, lab_src_url=https://github.com/vlead/continuous-integration.git, version=refactor, key=defaultkey, +2017-08-23 16:46:53,275: DEBUG: api.py: index():53: lab_id = test, lab_src_url=https://github.com/vlead/analytics-db, version=refactor, key=defaultkey, +2017-10-04 14:22:23,577: DEBUG: api.py: index():54: lab_id = test, lab_src_url=https://github.com/Virtual-Labs/computer-programming-responsive-iiith, version=master, key=defaultkey, +2017-10-04 14:23:48,182: DEBUG: api.py: index():54: lab_id = test, lab_src_url=https://github.com/Virtual-Labs/computer-programming-responsive-iiith, version=master, key=defaultkey, +2017-10-04 14:28:01,475: DEBUG: api.py: index():54: lab_id = test, lab_src_url=https://github.com/Virtual-Labs/computer-programming-responsive-iiith, version=master, key=defaultkey, +2017-11-13 07:52:31,392: DEBUG: api.py: index():54: lab_id = repear, lab_src_url=https://github.com/vlead/computer-programming-responsive, version=master, key=defaultkey, +2017-11-13 08:41:25,220: DEBUG: api.py: index():54: lab_id = test, lab_src_url=https://github.com/vlead/computer-programming-responsive, version=master, key=defaultkey, +2017-11-13 09:22:03,266: DEBUG: api.py: index():54: lab_id = test, lab_src_url=https://github.com/vlead/computer-programming-responsive, version=master, key=defaultkey, +2017-11-13 09:32:17,681: DEBUG: api.py: index():54: lab_id = test, lab_src_url=https://github.com/vlead/computer-programming-responsive, version=master, key=defaultkey, +2017-11-13 19:53:26,301: DEBUG: api.py: index():54: lab_id = i201, lab_src_url=0293, version=master, key=defaultkey, +2017-11-15 11:44:23,932: DEBUG: api.py: index():54: lab_id = test, lab_src_url=https://github.com/vlead/computer-programming-responsive, version=master, key=defaultkey, diff --git a/src/ads-web-app/requirem b/src/ads-web-app/requirem new file mode 100644 index 0000000..106f39b --- /dev/null +++ b/src/ads-web-app/requirem @@ -0,0 +1,36 @@ +ADS==2.2 +argparse==1.2.1 +backports-abc==0.5 +backports.ssl-match-hostname==3.5.0.1 +boto==2.48.0 +bzr==2.1.1 +certifi==2017.7.27.1 +chardet==3.0.4 +click==6.7 +Flask==0.12.2 +Flask-OAuthlib==0.9.4 +idna==2.5 +iniparse==0.3.1 +itsdangerous==0.24 +Jinja2==2.9.6 +MarkupSafe==1.0 +mercurial==1.4 +netaddr==0.7.19 +oauthlib==2.0.2 +ordereddict==1.2 +OVPL==0.1 +paramiko==1.7.5 +pycrypto==2.0.1 +pycurl==7.19.0 +pygpgme==0.1 +pymongo==3.4.0 +requests==2.18.2 +requests-oauthlib==0.8.0 +sh==1.12.14 +singledispatch==3.4.0.3 +six==1.10.0 +tornado==4.5.1 +urlgrabber==3.9.1 +urllib3==1.22 +Werkzeug==0.12.2 +yum-metadata-parser==1.1.2 diff --git a/src/ads-web-app/requirement.txt b/src/ads-web-app/requirement.txt new file mode 100644 index 0000000..e69de29 diff --git a/src/ads-web-app/templates/base.html b/src/ads-web-app/templates/base.html index b092c29..8a0bb69 100644 --- a/src/ads-web-app/templates/base.html +++ b/src/ads-web-app/templates/base.html @@ -9,6 +9,7 @@ +
diff --git a/src/ads-web-app/templates/index.html b/src/ads-web-app/templates/index.html index afc9c63..6dba6b9 100644 --- a/src/ads-web-app/templates/index.html +++ b/src/ads-web-app/templates/index.html @@ -27,4 +27,33 @@

{{ message }}

+
+
+ + + {% endblock %} diff --git a/src/httplogging/helper.py b/src/httplogging/helper.py index 662f14e..5f51327 100644 --- a/src/httplogging/helper.py +++ b/src/httplogging/helper.py @@ -6,6 +6,7 @@ import os from __init__ import * from utils.envsetup import EnvSetUp +from Queue import * env = EnvSetUp.Instance() used_loggers = {} @@ -47,6 +48,7 @@ def log(arguments): lineno = arguments['lineno'][0] name = arguments['name'][0] message = arguments['msg'][0] + qmsg.put_msg(message) funcName = arguments['funcName'][0] logger = get_logger(name) fmt_string = "%(levelname)s: %(pathname)s:%(lineno)-4s-> %(message)s" @@ -59,3 +61,16 @@ def log(arguments): record = logger.makeRecord(name, levelname, funcName, lineno, fmt_string, record_format_args, None) logger.handle(record) + +class Message(): + def __init__(self): + self.msg = Queue() + def get_msg(self): + if not self.msg.empty(): + return self.msg.get() + def put_msg(self,mesg): + self.msg.put(mesg) + def q_emp(self): + return self.msg.empty() + +qmsg = Message() diff --git a/src/httplogging/http_logging_server.py b/src/httplogging/http_logging_server.py index c9c3727..e9b9ffd 100644 --- a/src/httplogging/http_logging_server.py +++ b/src/httplogging/http_logging_server.py @@ -30,6 +30,15 @@ def post(self): args=(self.request.arguments,)) t.start() +class TestHandler(tornado.web.RequestHandler): + def get(self): + if helper.qmsg.q_emp(): + self.write("nth") + else: + self.write(helper.qmsg.get_msg()) + + def post(self): + pass class OtherHandler(tornado.web.RequestHandler): def get(self): @@ -43,6 +52,7 @@ def post(self): app = tornado.web.Application( handlers=[ (r"/log/.*", LogHandler), + (r"/test", TestHandler), (r"/.*", OtherHandler), ], debug=True)