diff --git a/.custom_wordlist.txt b/.custom_wordlist.txt index 23015ae..714eae1 100644 --- a/.custom_wordlist.txt +++ b/.custom_wordlist.txt @@ -41,6 +41,7 @@ Bionic's Blazingly bool boolean +borked BranchMergeProposal breakpoint browserconfig @@ -48,6 +49,7 @@ BrowserNotificationMessages brz BugSubscription bugtracker +BugTask BugWatch BugWatches bugzilla @@ -57,8 +59,13 @@ buildd builddeb bzr bzr's +callgrind Canonical's CanonicalUrlData +capitalization +CategoryTesting +CategoryTipsAndTricks +cd Centralized centralized cfg @@ -68,10 +75,12 @@ changelog changeset changesets charmcraft +chroot Cmd CMDLINE Cobol codebase +codebrowse Codehosting codehosting codeimport @@ -91,9 +100,11 @@ cryptographic cryptographically CSCVS css +ctl Ctrl customisable customisations +customizations customize CVE Dalia @@ -122,11 +133,15 @@ DecoratedResultSet DecoratedResultSets defense defense +del deps +deryck Desc dev +devpad dh dia +dir DirectDelivery directDelivery Distro @@ -152,7 +167,9 @@ el else's emphasizes emphazising +ENOSPC EnumCol +environ epydoc Epytext ExceptionGuidelines @@ -162,6 +179,7 @@ fastdowntime fastnodowntime favicons FK +fk flavor FooBar foofunc @@ -181,6 +199,7 @@ fti functiondef ganesha gangotri +GDB geoIP geoip getFeatureFlag @@ -196,6 +215,8 @@ hba hg hirsute's hostnames +hotfix +hotfixes howto HSTS html @@ -206,6 +227,7 @@ IBranchMergeProposal IBranchTarget ICanonicalUrlData ICodeReviewMessage +idx iframe iharness IMailDelivery @@ -225,8 +247,13 @@ IPv IPython IRangeFactory irc +IrcId +ircid IRCMeetings +IrcNickname +ircnickname iter +iterable Javadoc javascript JavascriptUnitTesting @@ -235,7 +262,10 @@ jenkins jinja js JScript +json JSONEncoder +KCacheGrind +kcachegrind keyring kiko kompare @@ -247,6 +277,7 @@ LaunchpadFormView LaunchpadObjectFactory LaunchpadPpa LaunchpadProductionStatus +LaunchpadStatementTracer LaunchpadView lazr libera @@ -256,6 +287,7 @@ listdir ListRangeFactory LivePatching LiveScript +LocationError logpoints logrotate lookup @@ -268,6 +300,7 @@ Lp's lpbuildbot lpbuildd lpci +LPCONFIG LPHowTo lpnet lpreview @@ -287,6 +320,8 @@ matic maximize mbox mboxMailer +Meinel's +meliae MemCache memcache milestoneoverlay @@ -297,14 +332,18 @@ minimize mmm MockIo mockups +moin mojo mozilla +mta natively NavigationMenu NavigationMenus newsampledata +nextval nfs NPM +normalized NTP NULLs né @@ -327,6 +366,8 @@ organizing orm os OSA +osageorang +osageorange OWASP OWASP's PackageBranchTarget @@ -341,10 +382,12 @@ pdb PersonBranchTarget pentested performant +PerLanguageStatisticsView pgbouncer pgSQL pgsql pipx +pkey plaintext png po @@ -360,6 +403,7 @@ poller POMsgID pooler portlets +postfix PostGreSQL PostgreSQL postgresql @@ -394,10 +438,12 @@ programmatically prometheus proname proxied +pstats psql px py pydoctor +pygdb pypa PyPI qa @@ -417,6 +463,7 @@ realizes ReleaseCycles repl repo +RequestLogging RESTful resultset ResultSets @@ -445,6 +492,7 @@ setupDTCBrowser setupRosettaExpertBrowser setuptools SFTP +SIGHUP simplejson simplestreams Slony @@ -470,6 +518,7 @@ sqlbuilder SQLObject SQLObject's SQLObjectResultSets +sqltrace SRE SREs srv @@ -477,7 +526,9 @@ SshConnectionMultiplexing SSO StagingServer standardized +stderr stdin +stdout stepto steve stg @@ -505,8 +556,10 @@ synchronize synchronizing syntax syntaxes +sysctl systemd TableRenamePatch +tal talisker TAs TeamEmail @@ -526,10 +579,13 @@ TestsFromChanges TestsStyleGuide testtools textarea +thusly TipsForReviewers TLS +tmp todrop traceback +tracebacks tradeoff tradeoffs Transactional @@ -537,10 +593,12 @@ TranslationGroup TranslationImportQueue TranslationMessage TranslationTemplateItem +traveled traversers triaged triaging tsearch +txt Ubuntu ubuntu UCT @@ -549,11 +607,14 @@ unauthorized uncheck Uncomment unittest +unittests unobvious unproxied unsuffixed untriaged untrusted +upd +updatable upstreams url urls @@ -573,12 +634,14 @@ VM VMs VPN VPOTExport +VSCode webapp webdav webhook webservice webservice's webservices +wget wgrant's whitespace wildcherry diff --git a/.wordlist.txt b/.wordlist.txt index c2abb02..fdf0303 100644 --- a/.wordlist.txt +++ b/.wordlist.txt @@ -9,6 +9,7 @@ Charmhub CLI conf contactuser +cronscripts datetime DateTime DBA diff --git a/custom_conf.py b/custom_conf.py index ed724c2..c4455ef 100644 --- a/custom_conf.py +++ b/custom_conf.py @@ -127,10 +127,7 @@ 'https://wiki.canonical.com/InformationInfrastructure/OSA/LPHowTo/ManualCdImageMirrorProber', # private 'Trunk/Glue', # needs update '/Background', - '/HowToUseCodehostingLocally', # needs update - 'Database/TableRenamePatch', # needs update - 'Debugging#Profiling%20page%20requests', # needs update - 'Debugging#Special%20URLs', # needs update + '/Concepts', # needs update 'JavascriptUnitTesting/MockIo', # needs update 'https://git.launchpad.net/launchpad-mojo-specs/tree/mojo-lp-git/services', # private 'https://wiki.canonical.com/InformationInfrastructure/OSA/LaunchpadProductionStatus', # private @@ -142,6 +139,7 @@ r'https://github\.com/canonical/fetch-operator*', # private 'https://git.launchpad.net/charm-launchpad-buildd-image-modifier/tree/files/scripts/setup-ppa-buildd', # private 'https://git.zx2c4.com/cgit/', # unfortunately very flaky + 'https://wiki.canonical.com/InformationInfrastructure/OSA/RequestLogging/LP/Cowboys', # private ] # Pages on which to ignore anchors diff --git a/explanation/charms.rst b/explanation/charms.rst index 9e09218..8e2019f 100644 --- a/explanation/charms.rst +++ b/explanation/charms.rst @@ -17,7 +17,7 @@ with one or more `units `_. The layers provide common code used by multiple charms. The specs are used with `Mojo `_ to coordinate whole deployments of multiple applications; they contain configuration of individual applications and -`integrations `_ between applications. +`integrations `_ between applications. Principles ========== @@ -71,7 +71,7 @@ Creating a Launchpad charm -------------------------- If you don't already have a suitable testbed, then see the `Juju tutorial -`_ for how to set one up; +`_ for how to set one up; you should use the non-Kubernetes approach here. Assuming you have a suitable testbed to deploy charms with Juju, you can follow diff --git a/explanation/code.rst b/explanation/code.rst index a30e955..1f4679a 100644 --- a/explanation/code.rst +++ b/explanation/code.rst @@ -31,8 +31,7 @@ other asynchronous jobs associated with them. The `codehosting overview diagram :attachment:../images/codehosting.png` summarises how some of these systems interact. -You can `run the codehosting system -locally `__. +You can :doc:`run the codehosting system locally <../how-to/codehosting-locally>`. We no longer put significant effort into bzr hosting beyond making sure it remains functional. The future of bzr is diff --git a/explanation/database-performance.rst b/explanation/database-performance.rst index fb66b14..4e4e2b7 100644 --- a/explanation/database-performance.rst +++ b/explanation/database-performance.rst @@ -102,8 +102,7 @@ EXPLAIN ANALYZE on staging and qastaging can be used by LOSAs, the TAs, and squad leads. If you want to see how a query is working on a GET page locally, try the -`++oops++ `__ and -`++profile++ `__ tools. +:doc:`++oops++ and ++profile++ tools <../how-to/debugging>`. ++profile++ reportedly works on staging and qastaging now too. Unfortunately, they sometimes do not work properly for POSTs, and can't diff --git a/explanation/live-patching.rst b/explanation/live-patching.rst index 957e4fd..3e6bb18 100644 --- a/explanation/live-patching.rst +++ b/explanation/live-patching.rst @@ -126,8 +126,7 @@ Table Renaming Because code is being deployed separately from database updates, we need to maintain a fully updateable working alias for the old table name to -the new table name. See -`Database/TableRenamePatch `__ for an example +the new table name. See :doc:`Rename Database Table <../how-to/rename-database-table>` for an example database patch. Adding columns diff --git a/how-to/apply-schema-changes.rst b/how-to/apply-schema-changes.rst index 98542e0..b10a7f8 100644 --- a/how-to/apply-schema-changes.rst +++ b/how-to/apply-schema-changes.rst @@ -40,3 +40,13 @@ please run the following command: .. code-block:: bash make -C database/schema test + +If you only want to apply one specific patch for testing purposes, you can +run the following command: + +.. code-block: bash + + psql launchpad_dev -1 -f .sql + +This comment will apply file (``-f``) ``.sql`` as a single +transaction (``-1``) to the database named ``launchpad_dev``. diff --git a/how-to/applying-a-cowboy-to-qastaging.rst b/how-to/applying-a-cowboy-to-qastaging.rst new file mode 100644 index 0000000..a423f4c --- /dev/null +++ b/how-to/applying-a-cowboy-to-qastaging.rst @@ -0,0 +1,79 @@ +============================== +Applying a cowboy to qastaging +============================== + +.. note:: + + A cowboy is a hotfix applied to an instance, avoiding a proper deployment + by applying code changes directly on an instance. + +.. caution:: + + A cowboy is a valuable tool to either apply urgent hotfixes or test things + quickly e.g. in a staging environment. + + As it avoids a proper deployment, it also comes with a couple of negative + side effects: + + - a regular deployment overwrites the cowboy + + - the state of the instance is not clearly defined in the current code + and/or deployment configuration + + - and probably some more reasons + + All in all a cowboy should be applied only in rare and extraordinary + circumstances. + +Before you apply a cowboy, please have a quick conversation with your team +where the reason for the cowboy is thoroughly discussed. + +Applying the cowboy +=================== + +#. SSH into bastion ``ssh launchpad-bastion-ps5`` and switch to the + ``stg-launchpad`` user by running ``sudo -iu stg-launchpad``. + +#. Run ``source ~/.mojorc.qastaging``. + +#. Run ``juju config launchpad-appserver build_label`` and take a note of the + currently applied git commit id. + +#. Checkout that commit id locally. + +#. Apply the planned changes to your local checkout and create a local patch + file via e.g. ``git diff > .patch``. + +#. Log into the appropriate unit where you want to apply the patch to. + In this example, we will apply the patch to the application server + (``launchpad-appserver``). Applying patches to e.g. cronscripts will differ + as we might not need to restart the application server. + + - Get a list of units you need to apply the patches to. For the application + server run ``juju status launchpad-appserver``. + As we run more than one instance, we need to apply the next steps for + each of the units. + + - ssh into one of the units via ``juju ssh launchpad-appserver/2`` + + - Change into the directory with the source code via + ``cd /srv/launchpad/code``. + + - Apply the patch via ``git apply .patch``. + + - Restart the system via ``sudo systemctl restart launchpad``. + + .. note:: + + The restart command occasionally times out. Just run it again until + it works. + + - As already mentioned, repeat the previous steps for all units to make + sure all units have your patch applied. + + +Related information +=================== + +Applying cowboys to production need to be performed by IS, see +https://wiki.canonical.com/InformationInfrastructure/OSA/RequestLogging/LP/Cowboys. diff --git a/how-to/codehosting-locally.rst b/how-to/codehosting-locally.rst new file mode 100644 index 0000000..c835ebc --- /dev/null +++ b/how-to/codehosting-locally.rst @@ -0,0 +1,141 @@ +How To Use Codehosting Locally +============================== + +**Note:** This guide is for bazaar hosting. For git hosting, see `the +turnip README `__. + +Just as it's possible to run the launchpad web application locally, it +is also possible to run the complete codehosting stack on your +development machine. The main awkwardness is that you have to manually +kick off some of the things that are usually done by cronjobs. + +Make sure Apache is set up +-------------------------- + +Various parts of the codehosting system require that Apache is +configured appropriately. If launchpad.test works at all for you, you've +probably already done this, but if it doesn't work, running + +:: + + sudo make install + +from the root of your launchpad tree should configure things +appropriately. + +Get things running +------------------ + +Getting all the servers that need to be running started is as simple as +running + +:: + + make run_all + +or + +:: + + make run_codehosting + +in the root of your Launchpad tree. + +Set up a user +------------- + +First ensure you have an mta installed e.g. postfix. If you run + +:: + + ./utilities/make-lp-user + +, you can use + +:: + + lp://test/ + +shortcuts. + +You can also use the 'mark' launchpad user, the only user in the sample +data with an ssh key set up, but it's probably best not to these days. + +Push up a branch +---------------- + +If you ran + +:: + + make-lp-user + +, this is just a few more keystrokes than pushing a branch to launchpad: + +:: + + bzr push -d lp://test/~/+junk/branchname + +You might have to add the following to ~/.ssh/config: + +:: + + Host bazaar.launchpad.test + Port 5022 + Hostname launchpad.test + +Pull and scan the branch +------------------------ + +At this point the branch is just in the 'hosted area', and needs to be +*scanned* (data about the branch copied into the Launchpad database). + +On production, this happens via the magic of cron. Locally you can make +it happen by running + +:: + + cronscripts/process-job-source.py IBranchScanJobSource + +. + +Now you have a fully working and up-to-date branch -- you should be able +to look at the branch page in Launchpad, view the source in codebrowse, +and so on. + +Troubleshooting +--------------- + +- If you have troubles pushing to a local code hosting instance with an + error like below, the stale /var/tmp/launchpad_forking_service.sock + might be the problem. Remove it and restart code hosting. + +.. raw:: html + + + +:: + + $ bzr push lp://test/~danilo/translated/trunk + exec request failed on channel 0 + bzr: ERROR: Connection closed: Unexpected end of message. Please check connectivity and permissions, and report a bug if problems persist. + +- If you are receiving connection refused to port 5022, in + \`configs/development/launchpad-lazr.conf`, under the + \`[codehosting]\` heading, amend the line: + +.. raw:: html + + + +:: + + port: tcp:5022:interface=127.0.0.88 + +to + +:: + + port: tcp:5022:interface=0.0.0.0 + +Alternatively, push from within the container. \ No newline at end of file diff --git a/how-to/debugging.rst b/how-to/debugging.rst new file mode 100644 index 0000000..ba08a17 --- /dev/null +++ b/how-to/debugging.rst @@ -0,0 +1,668 @@ +Debug Stories/Pagetests +======================= + +Debugging stories (a.k.a. pagetests) can be kind of a pain because they +build up state as they go. So if a test fails down deep in the doctest, +and you want to see what the page really looks like at that point, you'd +normally have to manually click through each step in the doctest until +you get to the place that's broken. + +But there's an easier way! + +Just add this at the point where your pagetest is failing: + +:: + + >>> stop() + +This is just a convenience wrapper around ``pdb.set_trace()`` except +that it also redirects stdout back to your console. When your pagetest +hits this line, you'll be dropped into the debugger. Now, go to a +different shell and type: + +:: + + % make LPCONFIG=testrunner run + +(*Note from deryck: To get the above make run command working, I had to +use the value in os.environ['LPCONFIG'] obtained at the pdb prompt, +rather than just "testrunner"*) + +This starts up the appserver, except that it will use the +``testrunner`` configuration. What's cool about that is that this +configuration attaches to the same database as the pagetest you're now +stopped inside of. Meaning, all that state your doctest has built up, is +available to your browser. So just hit the URL of the page that your +doctest is failing on, and see it in all it's wondrous, albeit borked, +glory. + +Additionally, it is also possible to connect directly to the database +the page test is using. + +:: + + % psql -d launchpad_ftest + +However, uncommitted transactions will not be visible in the database. +So if there is something particular you are trying to query, like a bug +subscription, you may need to add the following to the page test: + +:: + + >>> transaction.commit() + +Debugging memory leaks +---------------------- + +The app servers and the Librarian install a signal handler to dump their +memory using `meliae `__. To make use of +that you just have to send the 44 signal to the `appropriate +process `__. +This will create a file named /tmp/launchpad-memory.dump (or +/tmp/librarian-memory.dump, for the Librarian), which you can debug +later. + +In general, though, the leaks will happen in production, so the LOSAs +will give us a memory dump. Assuming you've got a memory.dump file, this +is how you load it: + +:: + + $ ./bin/py + >>> from meliae import loader + >>> om = loader.load('memory.dump') + >>> s = om.summarize(); s + +This will give you a nice summary of the kinds of objects that consume +the most memory. Something like: + +:: + + Total 704144 objects, 768 types, Total size = 129.5MiB (135823229 bytes) + Index Count % Size % Cum Max Kind + 0 108751 15 68190568 50 50 3146008 dict + 1 192992 27 16136424 11 62 6904 tuple + 2 162136 23 14850362 10 73 11103 str + ... + +You can then use that information to navigate the object tree as +described in `John Meinel's excellent +post `__, and +hopefully find what's leaking. + +Debugging With GDB +------------------ + +Some kinds of bugs (segfaults, hung processes, out-of-control daemons) +are hard to debug from within Python. See +`Debugging with GDB `__ for how to debug them. + +Debugging Core Dumps +-------------------- + +A quick sketch of how to read core files produced from production machines by IS: + +1. Have core dump files moved to osageorang +2. Ensure you have access to osageorange. You'll need to ping a member of IS on #launchpad-ops to get access; you cannot ssh from devpad to osageorange. +3. Get pygdb (lp:pygdb) in your $HOME on osageorang +4. Ssh to osageorange, and do: + +:: + + schroot -c lucid-cat-amd64 + +This puts you in a chroot with the same packages installed as on production + +5. Then cd into pygdb dir and do something like: + +:: + + python backtrace.py -c $PATH_TO_FILE/core.XXX > ~/core.XXX-out.txt + +6. Read output and profit! + + +Debugging Buildd with the Visual Studio Code IDE +------------------------------------------------ + +Although we `set up Buildd in a +VM `__, Unit +tests can be debugged in a visual & interactive way with Visual Studio +Code with below setup for remote ssh&debug. + +:: + + sudo snap install --classic code + +Launch VSCode and install the "Python" and "Remote - SSH" extensions. +For the Python extension perform the install on the remote host (your +Buildd VM) as well. + +F1 to open the command palette and type: + +:: + + Remote-SSH: Open SSH Configuration file + +From the drop-down choose your home ssh config file (~/.ssh/config - +if you don't have one create it and add your configuration). Add the +following entry to it (for Host I have the name of my Buildd VM and for +the IP of my Buildd VM): + +:: + + HostName + + +:: + + Host buildd + HostName xxx.xx.x.xxx + User ubuntu + IdentityFile ~/.ssh/buildd_rsa.pub + +F1 to open the command palette and type "Remote-SSH: Connect to Host..." +and choose "buildd" from the drop-down. You might be prompted for the +password for the "ubuntu" user. + +Once the ssh connection is established go to the Explorer and "Open +Folder" -> launchpad-buildd (the git clone of the buildd repo on your +VM). + +Configure the test framework (visual examples +`\here `__). +For buildd choose unittests for the lpbuildd folder and the test*.py file pattern. + +Tips: + +1. Command palette opens with F1. + +2. When the workspace is large and contains many files VS Code file +watcher is running out of handles (ENOSPC Error visible is you start +VSCode in terminal with): + +:: + + code --verbose + +Solution to this is: to see your current limit: + +:: + + cat /proc/sys/fs/inotify/max_user_watches + +Add this line: + +:: + + fs.inotify.max_user_watches=524288 + +to /etc/sysctl.conf and then: + +:: + + sudo sysctl -p + +. + +Special URLs +------------ + +Launchpad provides special URLs that can be used to help with debugging. + +\|\| *URL element* \|\| *Description* \|\| "Availability" \|\| \|\| +\``++debug++tal`\` \|\| show the TAL declarations *in the HTML source +code* \|\| developer box \|\| \|||||\| Example: +https://launchpad.test/++debug++tal \|\| \|\| \``++debug++source`\` \|\| +show path to templates for a given view *in the HTML source code* \|\| +developer box \|\| \|||||\| Example: +https://launchpad.test/++debug++source \|\| \|\| \``++profile++`\` \|\| +Get help on how to use the ++profile++ option. \|\| developer box, +[qa]staging \|\| \|||||\| Example: https://launchpad.test/++profile++ or +https://qastaging.launchpad.net/++profile++ \|\| \|\| +\``++profile++sql`\` \|\| See SQL queries used by the page. \|\| +developer box, [qa]staging \|\| \|||||\| Example: +https://launchpad.test/++profile++sql or +https://qastaging.launchpad.net/++profile++sql \|\| \|\| +\``++profile++sqltrace`\` \|\| See SQL queries and Python stack traces +that led to them. \|\| developer box, [qa]staging \|\| \|||||\| Example: +https://launchpad.test/++profile++sqltrace or +https://qastaging.launchpad.net/++profile++sqltrace \|\| \|\| +\``++profile++show`\` \|\| Show Python profile data and OOPS data, +including SQL queries and timing. \|\| developer box, [qa]staging \|\| +\|||||\| Example: https://launchpad.test/++profile++show or +https://qastaging.launchpad.net/++profile++show \|\| \|\| +\``++profile++pstats`\` \|\| Generate a pstats (Python standard library) +profile file on the file system. Browser page gives you full path to +generated file. **Note that, on [qa]staging, you will need to ask LOSAs +to get you the file.** \|\| developer box, [qa]staging \|\| \|||||\| +Example: https://launchpad.test/++profile++pstats or +https://qastaging.launchpad.net/++profile++pstats \|\| \|\| +\``++profile++callgrind`\` \|\| Generate a KCacheGrind profile file on +the file system. Browser page gives you full path to generated file. +**Note that, on [qa]staging, you will need to ask LOSAs to get you the +file.** \|\| developer box, [qa]staging \|\| \|||||\| Example: +https://launchpad.test/++profile++callgrind or +https://qastaging.launchpad.net/++profile++callgrind \|\| \|\| +\``++oops++`\` \|\| record an OOPS report while still rendering the page +correctly. The OOPS id is provided in the HTML source code \|\| ALL \|\| +\|||||\| Example: https://launchpad.test/++oops++ or +https://qastaging.launchpad.net/++oops++ \|\| \|\| \``++form++`\` \|\| +Not a debug tool. Used for JS. Gives inner form HTML. \|\| ALL \|\| +\|||||\| Example: https://launchpad.test/~/+edit/++form++ \|\| + +Some of those can combined, like: ``++debug++tal,source`` or +``++profile++show,pstats``. + +``++debug++errors`` is not working currently, probably because of +Launchpad customizations. It is supposed to show tracebacks of errors +handled in the template. + +Tracing SQL statements through STORM +------------------------------------ + +These can be useful when optimising pages to run fewer queries, as you +can see exactly when and what is executed rather than pulled from cache. + +Tracing a full request +~~~~~~~~~~~~~~~~~~~~~~ + +Set ``LP_DEBUG_SQL=1`` environment variable before running ``make +harness`` or ``make run`` to get the SQL statements as they are run, +along with the start and stop times and the name of the database on +which the statement was run. Note that in a request the times are +relative to the start of the request. For scripts and ``make harness``, +the start time is always 0 and the stop time is the duration of the SQL +call. + +Set ``LP_DEBUG_SQL_EXTRA=1`` to get all of the above, plus tracebacks +for every query execution, including template and traversal information. + +When using ``make run``, these affect all requests while the server is +running, and output the value in the console. + +Alternatively, to only look at a *single* request's values in the +browser, use ``++profile++sql`` instead, which includes the +information equivalent to ``LP_DEBUG_SQL=1``; or use +``++profile++sqltrace``, which gives you all of the information +equivalent to ``LP_DEBUG_SQL_EXTRA=1``. These are described above in +the "Special URLs" section. + +Tracing a part of a request +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``from storm.tracer import debug; debug(True)`` will cause all +statements run by Storm to be written to stderr. ``debug(False)`` +disables this behaviour. + +Alternatively, if you find ``LP_DEBUG_SQL=1`` and/or +``LP_DEBUG_SQL_EXTRA=1`` handy but want more control turning it on and +off within a request, in the debugger you can make sure the +``LaunchpadStatementTracer`` is the first in the results of +``get_tracers`` and modify as needed. For instance, you can do the +following. + +This gives output equivalent to ``LP_DEBUG_SQL=1`` but for only as +long as ``_debug_sql = True``. + +:: + + from storm.tracer import get_tracers + get_tracers()[0]._debug_sql = True + +This gives output equivalent to \``LP_DEBUG_SQL_EXTRA=1`\` but for only +as long as \``_debug_sql_extra = True``. + +:: + + from storm.tracer import get_tracers + get_tracers()[0]._debug_sql_extra = True + +Tracing a code snippet +~~~~~~~~~~~~~~~~~~~~~~ + +Similar to the previous section, sometimes you want to look at the SQL +of just a certain slice of code, such as within ``make harness``. The +``StormStatementRecorder`` can be a useful tool for this. + +Basic usage will get you the SQL run while the recorder is used: + +:: + + from lp.testing import StormStatementRecorder + + with StormStatementRecorder() as recorder: + ...code that touches the DB goes here... + + print recorder + +Printing the recorder gives you a full output of what happened. You can +also look at .statements, .count, and so on (use dir!). + +You can get all tracebacks by passing True when you instantiate the +recorder ( + +:: + + StormStatementRecorder(True) + +). Again, print the recorder to see the results. + +You can conditionally get tracebacks by passing a callable that receives +a SQL query string and returns a boolean True if a traceback should be +collected, and False if it should not. The SQL will be normalized to +capitalization and space normalized. For example, + +:: + + StormStatementRecorder(lambda sql: 'STRUCTURALSUBSCRIPTION' in sql) + +would get you tracebacks when the SQL has something to do with +structural subscriptions. + +Getting more information in your tracebacks +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The tracebacks from ``LP_DEBUG_SQL_EXTRA=1`` and +``++profile++sqltrace`` include extra information from page templates +and traversals to tell you the expressions and values being processed. +If you have functions or loops for which you'd also like to add your own +extra debugging information to the tracebacks, here is how. + +If you don't plan on checking the change in, or if the string you want +already exists and does not need to be generated, just assign the string +with the extra information you want to the variable name + +:: + + __traceback_info__ + +. That string will then be included in the information for that frame in +tracebacks generated by this machinery, as well as in renderings of +tracebacks from the appserver. + +If you plan on checking the change in, you should be more careful: we +only want to do the work if a traceback is rendered, not every time the +code path is traveled. Then you have two options. The first is to create +an object that will do the work only when it is cast to a string (in + +:: + + __str__ + +) and assign it to a variable named + +:: + + __traceback_info__ + +, as above. + +The second, more involved option is to assign a two-tuple to + +:: + + __traceback_supplement__ + +. The first element of the tuple should be a factory, and the second +argument should be an iterable that is passed to the factory as + +:: + + *args + +. The factory should produce an object with any or all of the following +attributes: + +| ``source_url:: Some string that represents a source.  For page templates, this is the path to the template file.`` +| ``line:: value castable to str that is presented as a line number.`` +| ``column:: value castable to str that is presented as a column number.`` +| ``expression:: value castable to str that is presented as an expression currently being processed (like a TALES expression).`` +| ``warnings:: an iterable of strings that represent some warning to communicate.`` +| ``getInfo:: a callable that returns some extra string.`` + +Tracing SQL statements with PostgreSQL +-------------------------------------- + +Statement logging can be configured in ``postgresql.conf``, by setting +``log_statement`` to one of ``none``, ``ddl``, ``mod`` or ``all`` +(`docs `__). +The server needs to be reloaded (by ``SIGHUP`` or ``pg_ctl reload``) for +changes to take effect. + +It can also be set for a session, user or database: + +\` SET log_statement TO 'all'; -- +\`(\ `docs `__) + +\` ALTER ROLE launchpad SET log_statement TO 'all'; -- +\`(\ `docs `__) + +\` ALTER DATABASE launchpad_dev SET log_statement TO 'all'; -- +\`(\ `docs `__) + +Once enabled, statements will be logged to +``/var/log/postgresql/postgresql-*-main.log``. + +<> + +Getting past "LocationError: 'json'" in TAL template tracebacks +--------------------------------------------------------------- + +If you're testing with a new TAL template (.pt file) and you get +nasty-looking tracebacks that says something about + +:: + + LocationError: (, 'json') + +then try visiting the corresponding URL in the web services API. For +example, if https://bugs.launchpad.net/launchpad gets an unwieldy +traceback, then try +https://launchpad.net/api/beta/launchpad instead; you'll often get a +*much* more comprehensible error +trace that way. + +Using iharness for digging error tracebacks +------------------------------------------- + +If you are reading this, most probably you have noticed that when things +get wrong, ZOPE and TAL will rather give you a pointless +**LocationError** without to much information about what is causing it. + +To find out what exactly went wrong you can use *make iharness* and +investigate that specific **LocationError** + +Let's say that you got this error for *language_translation_statistics*: + +:: + + LocationError: (, + 'language_translation_statistics') + +To start the testing/debugging environment (the harness) run: + +:: + + make iharness + +Next you will have to import your classed and get your object. In our +example we were trying to get the *!PerLanguageStatisticsView* for +*ubuntu['hoary']* series. + +:: + + from canonical.launchpad.webapp.servers import LaunchpadTestRequest + from lp.our.module import PerLanguageStatisticsView + + #create and initialize the view + ubuntu = getUtility(ILaunchpadCelebrities).ubuntu + view = PerLanguageStatisticsView (ubuntu['hoary'], LaunchpadTestRequest()) + view.initialize() + + #request the view key + key = view.language_translation_statistics + +Now you should see a more meaningful message. + +Profiling page requests +----------------------- + +You can generate +`KCacheGrind `__ and +pstats (Python standard library) profiles of requests on your local +system. + +On your developer machine, try going to +https://launchpad.test/++profile++ or +https://launchpad.test/++profile++/~mark/+archive/ppa . Inserting +++profile++ in the URL like this will give you instructions on how to +use the feature. + +The ++profile++ mechanism has a number of features now, as described in +the "Special URLs" section above. For Python profiling, it can generate +immediate profiles in the browser (++profile++show), profiles on the +filesystem ready for kcachegrind (++profile++callgrind), profiles on the +filesystem ready for pstats (++profile++pstats),or combinations (such as +++profile++show,pstats). + +If you want to use this on + +:: + + staging + +or + +:: + + qastaging + +, this is already set up for you. You may need to ask a LOSA to +temporarily increase the timeout for the page that you want to analyze +using the feature flags mechanism (e.g., if you want to profile +BugTask:+index pages, you'll need to ask LOSAs to add something like + +:: + + hard_timeout pageid:BugTask:+index 2 30000 + +to https://qastaging.launchpad.net/+feature-rules. That sets a timeout +of 30 seconds (30000 milliseconds). + +You can also turn on a configuration variable to profile *every* +request. Edit ``configs/development/launchpad-lazr.conf`` and add the +following section: + +:: + + [profiling] + profile_requests: True + +Then start the development server and make **ONE** request to the URL +you wish to profile (in order to make a single request on pages that +make subsequent JS calls immediately on load, you may need to use wget +or similar): + +:: + + $ make run + ... server starts... + $ curl -k https://launchpad.test/ -o /dev/null + # or + $ wget --no-check-certificate https://launchpad.test + +You can now load the resulting ``*.prof`` file into KCacheGrind + +:: + + $ kcachegrind 2010-07-20_10\:01\:46.680-RootObject\:index.html-OOPS-1662X1-Dummy-2.prof + +The doc for these features is lib/canonical/launchpad/doc/profiling.txt +, but you may find that the ++profile++ overlay gives you sufficient +instructions, if you use that approach. + +Profiling one part of your page +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you are working on a developer instance of Launchpad, you can also +insert calls directly in your code to profile certain code paths when +viewing pages. This will aggregate profiling calls within the request, +so you can do this around code that is called multiple times in the +request. Try something like this: + +:: + + from lp.services.profile import profiling + with profiling(): + # Do the work that you want profiled here! + +This will then generate a pstats file for you on the filesystem at the +end of the request, and give you the data in the browser as well. + +Debugging production OpenID problems +------------------------------------ + +You can use the production OpenID provider to debug problems that can't +be reproduced with the test provider by changing +configs/development/launchpad-lazr.conf thusly: + +:: + + [vhost.testopenid] + -hostname: testopenid.dev + +hostname: login.launchpad.net + +Debugging security proxy problems +--------------------------------- + +Ever wondered which attributes are protected on an instance and by which +permission? You can use debug_proxy to get the information you need. + +Example make harness session: + +:: + + francis@Casteneda:~/canonical/launchpad/bug-365098$ make harness + bin/harness + execute_zcml_for_scripts()... + Reading $PYTHONSTARTUP... + Initializing storm... + Creating the factory... + + >>> from lp.registry.interfaces.distribution import IDistributionSet + >>> ubuntu = getUtility(IDistributionSet).getByName('ubuntu') + >>> evolution = ubuntu.currentseries.getSourcePackage('evolution') + >>> from lazr.restful.debug import debug_proxy + >>> debug_proxy(evolution) + 'zope.security._proxy._Proxy (using zope.security.checker.Checker)\n + public: __eq__, __getitem__, __hash__, __ne__, _getOfficialTagClause, + all_bugtasks, bug_reported_acknowledgement, bug_reporting_guidelines, + bugtargetdisplayname, bugtargetname, bugtasks, closed_bugtasks, createBug, + critical_bugtasks, currentrelease, deletePackaging, development_version, + direct_packaging, displayname, distinctreleases, distribution, + distribution_sourcepackage, distroseries, enable_bugfiling_duplicate_search, + format, getBranch, getBranches, getBugCounts, getBugTaskWeightFunction, + getBuildRecords, getCurrentTemplatesCollection, getCurrentTranslationFiles, + getCurrentTranslationTemplates, getFirstEntryToImport, + getLatestTranslationsUploads, getMergeProposals, getPocketPath, + getSharingDetailPermissions, getSharingPartner, getSuiteSourcePackage, + getTemplatesAndLanguageCounts, getTemplatesCollection, + getTranslationImportQueueEntries, getTranslationTemplateByName, + getTranslationTemplateFormats, getTranslationTemplates, getUsedBugTags, + getUsedBugTagsWithOpenCounts, get_default_archive, has_bugtasks, + has_current_translation_templates, has_obsolete_translation_templates, + has_sharing_translation_templates, has_translation_files, + has_translation_templates, high_bugtasks, id, inprogress_bugtasks, + latest_published_component, latest_published_component_name, linkedBranches, + linked_branches, max_bug_heat, name, newCodeImport, new_bugtasks, + official_bug_tags, open_bugtasks, packaging, path, product, productseries, + published_by_pocket, recalculateBugHeatCache, releases, searchTasks, + setBranch, setMaxBugHeat, setPackaging, + setPackagingReturnSharingDetailPermissions, shouldimport, sourcepackagename, + summary, title, unassigned_bugtasks\n' + +-------------- + +CategoryTipsAndTricks CategoryTesting \ No newline at end of file diff --git a/how-to/index.rst b/how-to/index.rst index c2fafac..489982d 100644 --- a/how-to/index.rst +++ b/how-to/index.rst @@ -40,4 +40,3 @@ If you have a running instance of Launchpad, there are common tasks you might ne :maxdepth: 2 operating-launchpad - diff --git a/how-to/launchpad-development-tips.rst b/how-to/launchpad-development-tips.rst index 192fad6..e5c6e49 100644 --- a/how-to/launchpad-development-tips.rst +++ b/how-to/launchpad-development-tips.rst @@ -14,4 +14,7 @@ Launchpad development tips chameleon debug-tests-with-visual-studio-code debug-buildfarm-builder + debugging launchpad-api-docs + codehosting-locally + rename-database-table diff --git a/how-to/manage-users.rst b/how-to/manage-users.rst index 3442d10..d95ef71 100644 --- a/how-to/manage-users.rst +++ b/how-to/manage-users.rst @@ -39,19 +39,30 @@ For staging and qastaging we should: user by running ``sudo -iu stg-launchpad``. 2. SSH into the ``launchpad-admin`` juju unit, by running -``in-model qastaging juju ssh launchpad-scripts/leader``. +``in-model qastaging juju ssh launchpad-admin/leader``. -3. Switch to ``launchpad`` user, by running -``sudo su launchpad``. - -4. Add the user to the team by running: +3. Add the user to the team by running: .. code:: - LPCONFIG=launchpad-admin /srv/launchpad/code/utilities/anoint-team-member + /srv/launchpad/code/utilities/anoint-team-member In our example: .. code:: - LPCONFIG=launchpad-admin /srv/launchpad/code/utilities/anoint-team-member test-user admins + /srv/launchpad/code/utilities/anoint-team-member test-user admins + +.. note:: + + While running Launchpad scripts, it is often necessary to specify the + ``LPCONFIG`` environment variable with an appropriate value, for example + ``LPCONFIG=launchpad-admin``, before the invocation to use the right + Launchpad configuration for the current unit. This is not necessary here + because it is already set in the environment within the ``launchpad-admin`` + unit. Running this command in another unit, say ``launchpad-scripts``, + would require running it as + + .. code:: + + LPCONFIG=launchpad-scripts /srv/launchpad/code/utilities/anoint-team-member diff --git a/how-to/operating-launchpad.rst b/how-to/operating-launchpad.rst index dcc9e25..01cec2e 100644 --- a/how-to/operating-launchpad.rst +++ b/how-to/operating-launchpad.rst @@ -15,4 +15,5 @@ Operating Launchpad transfer-project-ownership create-bot-account porting-builders-to-newer-ubuntu-versions - building-lpdev-images + applying-a-cowboy-to-qastaging + building-lpdev-images \ No newline at end of file diff --git a/how-to/rename-database-table.rst b/how-to/rename-database-table.rst new file mode 100644 index 0000000..380e4f9 --- /dev/null +++ b/how-to/rename-database-table.rst @@ -0,0 +1,59 @@ +========================= +Renaming a Database Table +========================= + +Renaming a table involves a few steps: + +1. Rename the table +2. Rename dependent objects, like constraints, indexes, and sequences. +3. Create a writable view using the old name so deployed code keeps working. + +For example: + +.. code-block:: sql + + -- Copyright 2011 Canonical Ltd. + -- This software is licensed under the GNU Affero General Public License version 3 (see the file LICENSE). + SET client_min_messages=ERROR; + + -- Rename the table + ALTER TABLE IrcId RENAME TO IrcNickname; + + -- And rename sequences, constraints, and indexes to match. + ALTER SEQUENCE ircid_id_seq RENAME TO ircnickname_id_seq; + ALTER TABLE IrcNickname + -- With Slony 1.2.22, renaming the primary key fails. Leave these + -- with the old names for now. + -- DROP CONSTRAINT ircid_pkey, + -- ADD CONSTRAINT ircnickname_pkey PRIMARY KEY (id), + DROP CONSTRAINT ircid_person_fk, + ADD CONSTRAINT ircnickname_person_fk + FOREIGN KEY (person) REFERENCES Person; + ALTER INDEX ircid_person_idx RENAME TO ircnickname_person_idx; + + -- Create a view with the old table name for backwards compatibility. + CREATE OR REPLACE VIEW IrcId AS SELECT * FROM IrcNickname; + + -- Make the backwards compatibility view updatable. + CREATE OR REPLACE RULE IrcId_ins AS ON INSERT TO IrcId DO INSTEAD + INSERT INTO IrcNickname VALUES ( + COALESCE(NEW.id, nextval('ircnickname_id_seq')), + NEW.person, + NEW.network, + NEW.nickname) + RETURNING IrcNickname.*; + + CREATE OR REPLACE RULE IrcId_upd AS ON UPDATE TO IrcId DO INSTEAD + UPDATE IrcNickname SET + id = NEW.id, + person = NEW.person, + network = NEW.network, + nickname = NEW.nickname + WHERE id = OLD.id + RETURNING IrcNickname.*; + + CREATE OR REPLACE RULE IrcId_del AS ON DELETE TO IrcId DO INSTEAD + DELETE FROM IrcNickname WHERE id = OLD.id + RETURNING IrcNickname.*; + + INSERT INTO LaunchpadDatabaseRevision VALUES (2208, 85, 0); \ No newline at end of file diff --git a/reference/services/fetch-service.rst b/reference/services/fetch-service.rst index 6bc54a3..9768b2a 100644 --- a/reference/services/fetch-service.rst +++ b/reference/services/fetch-service.rst @@ -124,13 +124,19 @@ Related specifications (only accessible for Canonical employees) Log files --------- +See `FreshLogs documentation `_. + Production ~~~~~~~~~~ -Not set up. + +* ``rless fetch-service.lp.internal::fetch-service-logs/fetch-service.log`` Qastaging ~~~~~~~~~ -Currently, to access the fetch-service internal logs, one needs to: + +* ``rless fetch-service.qastaging.lp.internal::fetch-service-logs/fetch-service.log`` + +Alternatively, to access the fetch-service internal logs, one needs to: 1. SSH into Launchpad's bastion and switch to the following user: ``stg-lp-fetch-service-qastaging@launchpad-bastion-ps5``. @@ -141,7 +147,7 @@ Currently, to access the fetch-service internal logs, one needs to: 3. Run ``sudo snap logs fetch-service -n 100 -f`` (where ``-n`` sets the number of log lines, and ``-f`` keeps up the latest logs up-to-date). -Alternatively, you can also check the logs in the following directory: +You can also check the logs in the following directory: ``/var/snap/fetch-service/current``. Monitoring