refactor file structure

7 files changed, 558 insertions, 0 deletions

diff --git a/src/autodep/logfs/__init__.py b/src/autodep/logfs/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/autodep/logfs/__init__.py
diff --git a/src/autodep/logfs/fstracer.py b/src/autodep/logfs/fstracer.py
new file mode 100644
index 0000000..c724b61
--- /dev/null
+++ b/src/autodep/logfs/fstracer.py
@@ -0,0 +1,276 @@
+#!/usr/bin/env python2
+"""
+This module is a bridge between low-level logging services and high level handling dependency logic
+"""
+
+import os
+import sys
+import stat
+import time
+import tempfile
+import socket
+import select
+import re
+import signal
+
+import proc_helpers
+
+import logger_hooklib
+import logger_fusefs
+
+stop=False;
+stoptime=0;
+
+def parse_message(message):
+  ret=message.split("\0")
+  return ret
+
+# check if proccess is finished
+def checkfinished(pid):
+  if not os.path.exists("/proc/%d/stat" % pid):
+	return True
+
+  try:
+	pid_child,exitcode = os.waitpid(pid, os.WNOHANG)
+  except OSError, e:
+	if e.errno == 10: 
+	  return False
+	else:
+	  raise
+  
+  if pid_child==0:
+	return False
+  return True
+
+#check if process is zombie
+def iszombie(pid):
+  try:
+	statfile=open("/proc/%d/stat" % pid,"r")
+	line=statfile.readline()
+	statfile.close()
+	line=line.rsplit(")")[1] # find last ")" char
+	line=line.strip()
+	match=re.match(r"^(\w)",line)
+	if match==None:
+	  print "Failed to get check if process is zombie. Format of /proc/<pid>/stat is incorrect. Did you change a kernel?"
+	  return False
+	
+	return match.group(1)=="Z"
+	
+  except IOError,e:
+	return True
+  
+
+# uses /proc filesystem to get pid of parent
+# it is not used in program. Using function on C instead
+def getparentpid(pid):
+  try:
+	statfile=open("/proc/%d/stat" % pid,"r")
+	line=statfile.readline()
+	statfile.close()
+	line=line.rsplit(")")[1] # find last ")" char
+	line=line.strip()
+	match=re.match(r"^\w\s(\d+)",line)
+	if match==None:
+	  print "Failed to get parent process. Format of /proc/<pid>/stat is incorrect. Did you change a kernel?"
+	  return 1
+	 
+	return int(match.group(1))
+	
+  except IOError,e:
+	return 1
+
+#check if message came from one of a child
+def checkparent(parent,child):
+  #print "Parent %s, child %s"%(parent,child)
+  if child==1 or getparentpid(child)==1:
+	return True
+	  
+  currpid=child
+#   for(pid=getpid();pid!=0;pid=__getparentpid(pid))
+  while getparentpid(currpid)!=1:
+	currpid=getparentpid(currpid)
+	if currpid==parent:
+	  return True
+  
+  print "External actions with filesystem detected pid of external prog is %d" % child
+  return False
+
+
+# default access filter. Allow acess to all files
+def defaultfilter(eventname, filename, stage):
+  return True
+
+# run the program and get file access events
+def getfsevents(prog_name,arguments,approach="hooklib",filterproc=defaultfilter):
+  events={}
+  # generate a random socketname
+  tmpdir = tempfile.mkdtemp()
+  socketname = os.path.join(tmpdir, 'socket')
+
+  try:
+	sock_listen=socket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET)
+
+	sock_listen.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+	sock_listen.bind(socketname)
+	sock_listen.listen(64)
+	# enable connect a socket for anyone
+	os.chmod(tmpdir,stat.S_IRUSR|stat.S_IWUSR|stat.S_IXUSR|stat.S_IROTH|stat.S_IWOTH|stat.S_IXOTH)
+	os.chmod(socketname,stat.S_IRUSR|stat.S_IWUSR|stat.S_IXUSR|stat.S_IROTH|stat.S_IWOTH|stat.S_IXOTH)
+
+  except socket.error, e:
+    print "Failed to create a socket for exchange data with the logger: %s" % e
+    return []
+  else:
+	#print socketname
+	pid=os.fork()
+	if pid==0:
+	  logger=None
+	  if approach=="hooklib":
+		logger=logger_hooklib.logger(socketname)
+	  elif approach=="fusefs":
+		logger=logger_fusefs.logger(socketname)
+	  else:
+		print "Unknown logging approach"
+		sys.exit(1)
+	  
+	  logger.execprog(prog_name,arguments)
+	  
+	  # should not get here
+	  print "Launch likely was unsuccessful"
+	  sys.exit(1)
+	else:
+	  def signal_handler(sig, frame):
+		print "You pressed Ctrl+C!"
+		global stoptime
+		if(time.time()-stoptime>5):
+		  print "Sending SIGINT to child"
+		  print "Press again in 5 seconds to force exit"
+		  stoptime=time.time()
+		else:
+		  print "Sending SIGKILL to child"
+		  os.kill(pid,signal.SIGKILL)
+		  os._exit(1)
+	  global signal
+ 	  signal.signal(signal.SIGINT, signal_handler)
+ 	  
+	  epoll=select.epoll()
+	  epoll.register(sock_listen.fileno(), select.EPOLLIN)
+
+	  connects = 0;
+	  clients={}
+	  was_first_connect=False
+			  
+	  while True:
+		try:
+		  sock_events = epoll.poll(3)
+		  
+		  for fileno, sock_event in sock_events:
+			if fileno == sock_listen.fileno():
+			  #print "\n\nEVENT\n\n"
+			  ret = sock_listen.accept()
+			  #print ret
+			  if ret is None:
+			  #  print "\n\nPASS\n\n"
+				pass
+			  else:
+				(client,addr)=ret
+			  # print client
+				connects+=1; # client accepted
+				was_first_connect=True
+				epoll.register(client.fileno(), select.EPOLLIN)
+				clients[client.fileno()]=client
+				#print "opened %d" % client.fileno()
+			#elif sock_event & select.EPOLLHUP:
+			  #epoll.unregister(fileno)
+			  #clients[fileno].close()
+			  #del clients[fileno]
+			  #connects-=1
+			  
+			elif sock_event & select.EPOLLIN:
+			  s=clients[fileno]
+			  record=s.recv(8192)
+			  
+			  if not record: # if connection was closed
+				epoll.unregister(fileno)
+				clients[fileno].close()
+				del clients[fileno]
+				connects-=1
+				#print "closed %d"%fileno
+				continue
+			  
+			  message=record.split("\0")
+			  #if message[3]=="compile": #and message[1]=="debug":
+				#print message
+			  
+			  
+			  try:
+				if message[4]=="ASKING":
+				  if filterproc(message[1],message[2],message[3]):
+					#print "Allowing an access to %s" % message[2]
+					s.sendall("ALLOW\0"); # TODO: think about flush here
+					
+				  else:
+					print "Blocking an access to %s" % message[2]
+					s.sendall("DENY\0"); # TODO: think about flush here
+					
+				else:
+				  eventname,filename,stage,result=message[1:5]
+
+				  if not stage in events:
+					events[stage]=[{},{}]
+				  
+				  hashofsucesses=events[stage][0]
+				  hashoffailures=events[stage][1]
+				  
+				  if result=="OK":
+					if not filename in hashofsucesses:
+					  hashofsucesses[filename]=[False,False]
+					
+					readed_or_writed=hashofsucesses[filename]
+					
+					if eventname=="read":
+					  readed_or_writed[0]=True
+					elif eventname=="write":
+					  readed_or_writed[1]=True
+					  
+				  elif result[0:3]=="ERR" or result=="DENIED":
+					if not filename in hashoffailures:
+					  hashoffailures[filename]=[False,False]
+					notfound_or_blocked=hashoffailures[filename]
+					
+					if result=="ERR/2":
+					  notfound_or_blocked[0]=True
+					elif result=="DENIED":
+					  notfound_or_blocked[1]=True
+
+				  else:
+					print "Error in logger module<->analyser protocol"
+				  
+			  except IndexError:
+				print "IndexError while parsing %s"%record
+		except IOError, e:
+		  if e.errno!=4: # handling "Interrupted system call" errors
+			raise
+
+		if was_first_connect and connects==0:
+		  break
+
+		if len(sock_events)==0 and len(clients)==0: 
+		#  # seems like there is no connect
+		  print "It seems like a logger module was unable to start or failed to finish\n" + \
+				"Check that you are not launching a suid program under non-root user."
+		  return []
+		if len(clients)==0 and iszombie(pid):
+		  break
+
+	  epoll.unregister(sock_listen.fileno())
+	  epoll.close()
+	  sock_listen.close()
+
+	  _, exit_status=os.waitpid(pid,0)
+	  signal,exit_status=exit_status%256,exit_status/256
+	  print "Signal: %s Exit status: %s" % (signal,exit_status)
+  
+  return events
+
diff --git a/src/autodep/logfs/helpers/Makefile b/src/autodep/logfs/helpers/Makefile
new file mode 100644
index 0000000..370fe47
--- /dev/null
+++ b/src/autodep/logfs/helpers/Makefile
@@ -0,0 +1,7 @@
+all: proc_helpers.so
+
+proc_helpers.so: proc_helpers.c
+	$(CC) -shared -fPIC -Wall `python-config --cflags` `python-config --ldflags` $(CFLAGS) $(LDFLAGS) proc_helpers.c -o ../proc_helpers.so
+
+clean:
+	rm -f ../proc_helpers.so
diff --git a/src/autodep/logfs/helpers/proc_helpers.c b/src/autodep/logfs/helpers/proc_helpers.c
new file mode 100644
index 0000000..df20eef
--- /dev/null
+++ b/src/autodep/logfs/helpers/proc_helpers.c
@@ -0,0 +1,59 @@
+#include <Python.h>
+
+#define MAXPATHLEN 256
+#define MAXFILEBUFFLEN 2048
+
+
+static PyObject* py_getparentpid(PyObject* self, PyObject* args)
+{
+  int ok;
+  int pid;
+  ok=PyArg_ParseTuple(args, "i", &pid);
+  if(!ok)
+    return Py_BuildValue("i", 0);
+
+  char filename[MAXPATHLEN];
+  snprintf(filename,MAXPATHLEN, "/proc/%d/stat",pid);
+  FILE *stat_file_handle=fopen(filename,"r");
+  if(stat_file_handle==NULL)
+    return Py_BuildValue("i", 0);
+  
+  char filedata[MAXFILEBUFFLEN];
+  size_t bytes_readed=fread(filedata,sizeof(char),MAXFILEBUFFLEN,stat_file_handle);
+  if(bytes_readed==0 || bytes_readed>=MAXFILEBUFFLEN) {
+	fclose(stat_file_handle);
+	return Py_BuildValue("i", 0);
+  }
+  
+  filedata[bytes_readed]=0;
+  
+  char *beg_scan_offset=rindex(filedata,')');
+  if(beg_scan_offset==NULL) {
+	fclose(stat_file_handle);
+	return Py_BuildValue("i", 0);
+  }
+  
+  pid_t parent_pid;
+  int tokens_readed=sscanf(beg_scan_offset,") %*c %d",&parent_pid);
+  if(tokens_readed!=1) {
+	fclose(stat_file_handle);
+	return Py_BuildValue("i", 0);
+  }
+  fclose(stat_file_handle);
+  
+  if(pid==1)
+	return Py_BuildValue("i", 0); // set this explicitly. 
+					     // I am not sure that ppid of init proccess is always 0
+  
+  return Py_BuildValue("i", parent_pid);
+}
+
+static PyMethodDef proc_helpers_methods[] = {
+	{"getparentpid", py_getparentpid, METH_VARARGS},
+	{NULL, NULL}
+};
+
+void initproc_helpers()
+{
+	(void) Py_InitModule("proc_helpers", proc_helpers_methods);
+}
diff --git a/src/autodep/logfs/logger_fusefs.py b/src/autodep/logfs/logger_fusefs.py
new file mode 100644
index 0000000..6c135e8
--- /dev/null
+++ b/src/autodep/logfs/logger_fusefs.py
@@ -0,0 +1,132 @@
+#!/usr/bin/env python2
+
+import subprocess
+import time
+import os
+import sys
+import signal
+
+
+
+class logger:
+  socketname=''
+  readytolaunch=False
+  mountlist=["/dev/","/dev/pts","/dev/shm","/proc/","/sys/"]
+  rootmountpath="/mnt/logfs_root_"+str(time.time())+"/"
+  currpid=-1
+#  rootmountpath="/mnt/logfs_roott_"+"/"
+  
+  def __init__(self, socketname, accuracy=False):
+	if os.geteuid() != 0:
+	  print "only root user can use FUSE approach for logging"
+	  exit(1)
+
+	self.socketname=socketname
+	self.currpid=os.getpid()
+
+	if accuracy==False:
+	  self.mountlist=self.mountlist+["/lib64/", "/lib32/","/var/tmp/portage/"]
+	
+	
+	print "mounting root filesystem into %s. External access to this folder will be blocked. Please, DON'T DELETE THIS FOLDER !!" % self.rootmountpath
+	
+	try:
+	  os.mkdir(self.rootmountpath)
+	except OSError,e:
+	  if e.errno==17: # 17 is a "file exists" errno
+		pass			# all is ok
+	  else:
+		print "failed to make mount directory %s: %s" % (self.rootmountpath,e)
+		print "this error is fatal"
+		exit(1)
+
+
+	ret=subprocess.call(['mount','-o','bind','/',self.rootmountpath])
+	if ret!=0:
+	  print "failed to bind root filesystem to %s. Check messages above"%self.rootmountpath
+	  exit(1)
+	
+	os.environ["LOG_SOCKET"]=self.socketname
+	os.environ["PARENT_PID"]=str(self.currpid)
+
+	# TODO: change
+	ret=subprocess.call(['/home/bay/gsoc/logger/src/hook_fusefs/hookfs',self.rootmountpath,
+						 '-o','allow_other,suid'])
+	if ret!=0:
+	  print "failed to launch FUSE logger. Check messages above"
+	  exit(1)
+	
+	for mount in self.mountlist:
+	  if not os.path.exists(mount):
+		continue
+	  
+	  ret=subprocess.call(['mount','-o','bind',mount,self.rootmountpath+mount])
+	  if ret!=0:
+		print "failed to mount bind %s directory to %s. Check messages above" % (
+			   mount,self.rootmountpath)
+		exit(1)
+	self.readytolaunch=True;
+	
+  def __del__(self):
+	#we will delete the object manually after execprog
+	pass
+  
+  # launches command, if it returns not 0 waits for 1 or 2 second and launches again
+  # for various umounts
+  def smartcommandlauncher(self,args):
+	for waittime in (1,1,2):
+	  ret=subprocess.call(args)
+	  if ret==0:
+		return
+	  print "Auto-retrying after %d sec" % waittime
+	  time.sleep(waittime)
+	print "Giving up. Command %s failed" % args
+	
+  
+  def execprog(self,prog_name,arguments):
+	if self.currpid!=os.getpid():
+	  print "Detected an attempt to execute execproc in other thread"
+	  sys.exit(1)
+
+	pid=os.fork()
+	if pid==0:
+	  try:
+		cwd=os.getcwd()
+		os.chroot(self.rootmountpath)
+		os.chdir(cwd)
+		
+		env=os.environ.copy()
+		env["LOGGER_PROCESS_IS_INTERNAL"]="YES"
+
+		os.execvpe(prog_name, arguments, env)
+		sys.exit(1)
+	  except OSError, e:
+		print "Error while chrooting and starting a program: %s" % e
+		sys.exit(1)
+	  
+	else:
+	  exitcode=2; # if ctrl-c pressed then returning this value
+	  needtokillself=False
+	  try:
+		exitcode=os.wait()[1]/256;
+	  except KeyboardInterrupt:
+		needtokillself=True
+	  try:
+		print "Unmounting partitions"
+		self.mountlist.reverse()
+		for mount in self.mountlist:
+	  	  if not os.path.exists(self.rootmountpath+mount):
+			continue
+		  self.smartcommandlauncher(['umount','-l',self.rootmountpath+mount])
+		self.smartcommandlauncher(['fusermount','-z','-u',self.rootmountpath]);
+		self.smartcommandlauncher(['umount','-l',self.rootmountpath]);
+		os.rmdir(self.rootmountpath)
+
+	  except OSError, e:
+		print "Error while unmounting fuse filesystem: %s" % e
+		sys.exit(1)
+		
+	  if needtokillself: # we kill self for report the status correct
+		signal.signal(signal.SIGINT,signal.SIG_DFL)
+		os.kill(os.getpid(),signal.SIGINT)
+	  os._exit(exitcode)
diff --git a/src/autodep/logfs/logger_hooklib.py b/src/autodep/logfs/logger_hooklib.py
new file mode 100644
index 0000000..008fc56
--- /dev/null
+++ b/src/autodep/logfs/logger_hooklib.py
@@ -0,0 +1,23 @@
+#!/usr/bin/env python2
+
+import os
+import sys
+
+class logger:
+  socketname=''
+  hooklibpath='/home/bay/gsoc/logger/src/hook_lib/file_hook.so' # TODO: change
+  
+  def __init__(self,socketname):
+	self.socketname=socketname
+	
+  def execprog(self,prog_name,arguments):
+	try:
+	  env=os.environ.copy()
+	  env["LD_PRELOAD"]=self.hooklibpath
+	  env["LOG_SOCKET"]=self.socketname
+	  
+	  os.execvpe(prog_name, arguments, env)
+	except OSError, e:
+	  print "Failed to launch the programm: %s" % e
+	  sys.exit(1)
+
diff --git a/src/autodep/logfs/portage_utils.py b/src/autodep/logfs/portage_utils.py
new file mode 100644
index 0000000..a03f6af
--- /dev/null
+++ b/src/autodep/logfs/portage_utils.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python2
+
+import os
+import subprocess
+import re
+
+def getpackagesbyfiles(files):
+  ret={}
+  listtocheck=[]
+  for f in files:
+	if os.path.isdir(f):
+	  ret[f]="directory"
+	else:
+	  listtocheck.append(f)
+	  
+  try:
+	proc=subprocess.Popen(['qfile']+['--nocolor','--exact','','--from','-'],
+	  stdin=subprocess.PIPE, stdout=subprocess.PIPE,stderr=subprocess.PIPE, 
+	  bufsize=4096)
+	
+	out,err=proc.communicate("\n".join(listtocheck))
+	if err!=None:
+	  print "Noncritical error while launch qfile %s"%err;
+
+	lines=out.split("\n")
+	#print lines
+	line_re=re.compile(r"^([^ ]+)\s+\(([^)]+)\)$")
+	for line in lines:
+	  if len(line)==0:
+		continue
+	  match=line_re.match(line)
+	  if match:
+		ret[match.group(2)]=match.group(1)
+	  else:
+		print "Util qfile returned unparsable string: %s" % line
+
+  except OSError,e:
+	print "Error while launching qfile: %s" % e
+	
+	
+  return ret
+  
+def getfilesbypackage(packagename):
+  ret=[]
+  try:
+	proc=subprocess.Popen(['qlist']+['--nocolor',"--obj",packagename],
+	  stdout=subprocess.PIPE,stderr=subprocess.PIPE, 
+	  bufsize=4096)
+	
+	out,err=proc.communicate()
+	if err!=None and len(err)!=0 :
+	  print "Noncritical error while launch qlist: %s" % err;
+	
+	ret=out.split("\n")
+	if ret==['']:
+	  ret=[]
+  except OSError,e:
+	print "Error while launching qfile: %s" % e
+
+  return ret
+