Difference between revisions of "Online builds 10-apr-2013"

From GlueXWiki
Jump to: navigation, search
m
m
 
(35 intermediate revisions by the same user not shown)
Line 1: Line 1:
−
= File structure for release area =
+
= Release area and SVN file structures =
  
 
* Below is a proposed release directory structure.
 
* Below is a proposed release directory structure.
−
* Official release area is /gluex/builds.
+
* Official release area on HDCC is /gluex/builds.
 
* Release area only writeable from hdsys account, read-only from all other accounts.
 
* Release area only writeable from hdsys account, read-only from all other accounts.
−
* Package directories are what gets checked into SVN.
+
* Package directories are what gets checked into SVN (except jar and Linux dirs).
−
* Packages can be checked in/out by anyone.
+
* Packages can be checked in/out by anyone (but not from hdops account).
 
* Anyone can build and install packages in their own area.
 
* Anyone can build and install packages in their own area.
 
* Only hdsys can release a package into official release area.
 
* Only hdsys can release a package into official release area.
 +
* Using osrelease.pl (from offline) to define architecture/compiler directory names.
 
* Notes:
 
* Notes:
 
** Should SCONS dir be under packages?  Should it be installed?
 
** Should SCONS dir be under packages?  Should it be installed?
−
** Need to develop scripts to check out and build everything in one command.
+
** Need make_online_package script to set up dirs for new package.
−
 
+
−
 
+
  
−
  -- builds
+
  -- /gluex/builds
 
     `-- devel
 
     `-- devel
 
         |-- doc
 
         |-- doc
Line 49: Line 49:
  
  
−
= SCONS Scripts =
+
= SCONS scripts =
  
−
* Need one SConstruct file at the top of all packages.
+
* Need one SConstruct file in top directory of every package.
−
* SConstruct the same for ALL packages.
+
* SConstruct should be the same for ALL packages (can be modified)
 
* Need SConscript file in each libsrc,binsrc,scripts,test directory.
 
* Need SConscript file in each libsrc,binsrc,scripts,test directory.
 
* SConscript needed to define package dependencies (e.g. determine include paths for compiling).
 
* SConscript needed to define package dependencies (e.g. determine include paths for compiling).
−
* Need one Python script per package to define dependencies, where to find header files and libs, etc.
+
* Need one system-level Python script per package to define dependencies, where to find header files and libs, etc.
 +
* Define package root dirs via env vars $PACKAGE or $PACKAGEROOT (e.g. $EVIOROOT).
 +
* Note:  always compiles with -g (takes a little extra room but no performance penalty).
 +
* Have not thought about Java yet (Ant or SCONS)
  
  
 +
 +
* To build (there are many other options):
 +
$ scons                # build
 +
$ scons install        # install to $INSTALL_DIR if defined, otherwise install locally
 +
$ scons -c              # undo build
 +
$ scons -c install      # undo install
 +
$ scons debug=y        # link with -Og
 +
# scons opt=3          # link with -O3
 +
 +
 +
* Parallel Builds
 +
** I have not considered parallel builds of the entire online software system, and am not sure if SCONS can take care of this.
 +
** May require a custom script that partially parallelizes the full build, or maybe each package needs an SConscript file instead of SConstruct, with a global SConstruct above.
 +
** Current goal is to get everything working at the package level so developers can start creating new packages. 
 +
 +
 +
 +
'''SConstruct - one per package, same for all packages:'''
 +
 +
#  Generic SConstruct for Hall D online packages
 +
#
 +
#  ejw, 8-may-2013
 +
 +
 +
from halld_lib import init_halld_env, build_halld
 +
 +
 +
#  create standard environment based on command line variables
 +
env = init_halld_env(Variables())
 +
 +
 +
#  modify environment if desired
 +
 +
 +
#  build everything based on environment
 +
build_halld(env)
 +
 +
 +
 +
'''SConscript to build a library:'''
  
 
  #  SConscript for libsrc
 
  #  SConscript for libsrc
Line 66: Line 109:
 
    
 
    
 
   
 
   
−
  Import('*')
+
  Import('libEnv')
−
  from init_env import modify_environment
+
  from halld_lib import define_dependencies, buildLib
−
from util import buildLib
+
 
   
 
   
 
   
 
   
Line 80: Line 122:
  
  
 +
 +
'''SConscript to build binaries:'''
  
 
  #  SConscript for binsrc
 
  #  SConscript for binsrc
Line 88: Line 132:
 
   
 
   
 
   
 
   
−
  Import('*')
+
  Import('binEnv')
−
  from init_env import modify_environment
+
  from halld_lib import define_dependencies, buildBin
−
from util import buildBin
+
 
   
 
   
 
   
 
   
Line 102: Line 145:
  
  
−
Python script to define cMsg dependencies:
+
'''System-level Python script to define e.g. codaObject dependencies:'''
  
−
from util import check_if_dir_exist
 
 
  import os
 
  import os
 
   
 
   
−
def loadcmsg(env) :
+
  def loadcodaobject(env) :
 
  OSENV = os.environ
 
  OSENV = os.environ
−
CMSG = OSENV['CMSG']
 
−
check_if_dir_exist('CMSG')
 
−
if env['SHOWENV'] == "1":
 
−
print "Loading CMSG software from ", CMSG
 
 
   
 
   
−
cmsgincs = []
+
        if(OSENV.has_key('CODAOBJECTROOT') and os.path.exists(OSENV['CODAOBJECTROOT'])==True):
−
cmsgincs.append(CMSG + '/include')
+
                rootDir = OSENV['CODAOBJECTROOT']
−
env.AppendUnique(CPPPATH=cmsgincs)
+
        elif(OSENV.has_key('CODAOBJECT') and os.path.exists(OSENV['CODAOBJECT'])==True):
−
+
                rootDir = OSENV['CODAOBJECT']
−
cmsgldir = []
+
        else:
−
cmsgldir.append(CMSG + '/lib')
+
                print "?Need to set CODAOBJECTROOT or CODAOBJECT env variable!"
−
env.AppendUnique(LIBPATH = cmsgldir)
+
                sys.exit(1)
−
+
 
−
cmsglibs = []
+
        if env['SHOWENV'] == "1":
−
cmsglibs.append('cmsgxx')
+
                print "Loading CODAOBJECT software from ", rootDir
−
cmsglibs.append('cmsg')
+
 
−
cmsglibs.append('cmsgRegex')
+
 
−
cmsglibs.append('pthread')
+
#  include files
−
cmsglibs.append('rt')
+
        codaobjectincs = []
−
cmsglibs.append('dl')
+
        codaobjectincs.append(rootDir + '/include')
 +
        env.AppendUnique(CPPPATH=codaobjectincs)
 +
 
 +
 
 +
#  library directories
 +
        codaobjectldir = []
 +
        codaobjectldir.append(rootDir + '/lib')
 +
        env.AppendUnique(LIBPATH = codaobjectldir)
 +
 
 
   
 
   
−
  env.AppendUnique(LIBS = cmsglibs)
+
#  libraries
 +
        codaobjectlibs = []
 +
        codaobjectlibs.append('codaObject')
 +
        codaobjectlibs.append('pthread')
 +
        codaobjectlibs.append('rt')
 +
        codaobjectlibs.append('dl')
 +
        env.AppendUnique(LIBS = codaobjectlibs)

Latest revision as of 10:58, 8 May 2013

Release area and SVN file structures

  • Below is a proposed release directory structure.
  • Official release area on HDCC is /gluex/builds.
  • Release area only writeable from hdsys account, read-only from all other accounts.
  • Package directories are what gets checked into SVN (except jar and Linux dirs).
  • Packages can be checked in/out by anyone (but not from hdops account).
  • Anyone can build and install packages in their own area.
  • Only hdsys can release a package into official release area.
  • Using osrelease.pl (from offline) to define architecture/compiler directory names.
  • Notes:
    • Should SCONS dir be under packages? Should it be installed?
    • Need make_online_package script to set up dirs for new package.


-- /gluex/builds
   `-- devel
       |-- doc
       |-- jar
       |-- Linux_RHEL6-x86_64-gcc4.8.0
       |   |-- bin
       |   |-- include
       |   |-- lib
       |   `-- scripts
       |-- packages
       |   |-- aPackage
       |   |-- anotherPackage
       |   `-- cMsg2et
       |       |-- doc
       |       |-- java
       |           `-- org
       |               `-- jlab
       |                   `-- halld
       |       |-- jar
       |       |-- Linux_RHEL6-x86_64-gcc4.8.0
       |       |   |-- bin
       |       |   |-- include
       |       |   |-- lib
       |       |   `-- scripts
       |       `-- src
       |           |-- binsrc
       |           |-- examples
       |           |-- libsrc
       |           |   `-- cMsg2et -> .
       |           |-- scripts
       |           `-- test
       `-- SCONS


SCONS scripts

  • Need one SConstruct file in top directory of every package.
  • SConstruct should be the same for ALL packages (can be modified)
  • Need SConscript file in each libsrc,binsrc,scripts,test directory.
  • SConscript needed to define package dependencies (e.g. determine include paths for compiling).
  • Need one system-level Python script per package to define dependencies, where to find header files and libs, etc.
  • Define package root dirs via env vars $PACKAGE or $PACKAGEROOT (e.g. $EVIOROOT).
  • Note: always compiles with -g (takes a little extra room but no performance penalty).
  • Have not thought about Java yet (Ant or SCONS)


  • To build (there are many other options):
$ scons                 # build
$ scons install         # install to $INSTALL_DIR if defined, otherwise install locally
$ scons -c              # undo build
$ scons -c install      # undo install
$ scons debug=y         # link with -Og
# scons opt=3           # link with -O3


  • Parallel Builds
    • I have not considered parallel builds of the entire online software system, and am not sure if SCONS can take care of this.
    • May require a custom script that partially parallelizes the full build, or maybe each package needs an SConscript file instead of SConstruct, with a global SConstruct above.
    • Current goal is to get everything working at the package level so developers can start creating new packages.


SConstruct - one per package, same for all packages: 

#  Generic SConstruct for Hall D online packages
#
#  ejw, 8-may-2013


from halld_lib import init_halld_env, build_halld


#  create standard environment based on command line variables
env = init_halld_env(Variables())


#  modify environment if desired


#  build everything based on environment
build_halld(env)


SConscript to build a library: 

#  SConscript for libsrc
#
#  to customize fill in package dependencies
#
#  ejw, 16-apr-2013
 

Import('libEnv')
from halld_lib import define_dependencies, buildLib


#  define dependencies on other packages
define_dependencies(libEnv,'cmsg')



#  build the lib
buildLib(libEnv)


SConscript to build binaries: 

#  SConscript for binsrc
#
#  to customize fill in package dependencies
#
#  ejw, 16-apr-2013


Import('binEnv')
from halld_lib import define_dependencies, buildBin


#  define dependencies on other packages
define_dependencies(binEnv,'codaobject cmsg evio et')


#  build programs
buildBin(binEnv)


System-level Python script to define e.g. codaObject dependencies: 

import os

 def loadcodaobject(env) :
	OSENV = os.environ

        if(OSENV.has_key('CODAOBJECTROOT') and os.path.exists(OSENV['CODAOBJECTROOT'])==True):
                rootDir = OSENV['CODAOBJECTROOT']
        elif(OSENV.has_key('CODAOBJECT') and os.path.exists(OSENV['CODAOBJECT'])==True):
                rootDir = OSENV['CODAOBJECT']
        else:
                print "?Need to set CODAOBJECTROOT or CODAOBJECT env variable!"
                sys.exit(1)
 
        if env['SHOWENV'] == "1":
                print "Loading CODAOBJECT software from ", rootDir
 
 
#  include files
        codaobjectincs = []
        codaobjectincs.append(rootDir + '/include')
        env.AppendUnique(CPPPATH=codaobjectincs)
  
 
#  library directories
        codaobjectldir = []
        codaobjectldir.append(rootDir + '/lib')
        env.AppendUnique(LIBPATH = codaobjectldir)
 

#  libraries
        codaobjectlibs = []
        codaobjectlibs.append('codaObject')
        codaobjectlibs.append('pthread')
        codaobjectlibs.append('rt')
        codaobjectlibs.append('dl')
        env.AppendUnique(LIBS = codaobjectlibs)
Retrieved from "https://halldweb.jlab.org/wiki/index.php?title=Online_builds_10-apr-2013&oldid=47124"