Erlang Programming Exercise 12-3: The Database Server as an Application

First create an OTP Application directory structure. The work flow resembles something like the following if you’re on UNIX.

cd ~/work/erlang
mkdir ~/work/erlang/chapter12/{src,ebin,include,priv}
cp db_server_otp.erl db_server_sup.erl ~/work/erlang/chapter12/src
cd ~/work/erlang/chapter12/src
erlc *.erl
mv *.beam ../ebin

With the source code and BEAM files ready, now create an app file, db_server_app.erl:

-module(db_server_app).
-export([start/2,stop/1]).
-behavior(application).

start(_Type, _StartArgs) ->
    db_server_sup:start().

stop(_State) ->
    ok.

Compile this code and then move the BEAM file to the ebin folder. Create an application resource file describing the application, db_server.app in the ebin folder:

{application,
 db_server,
 [{description, "Erlang Prgramming Ch. 12 DB Server"},
  {vsn, "1.0"},
  {modules, [db_server_otp, db_server_sup, db_server_app]},
  {registered, [db_server_sup]},
  {applications, [kernel, stdlib]},
  {env, []},
  {mod, {db_server_app, []}}
 ]
}.

The final directory structure should look like this:

~/work/erlang/chapter12$ ls -1
ebin
include
priv
src
~/work/erlang/chapter12$ ls -1 ebin
db_server.app
db_server_app.beam
db_server_otp.beam
db_server_sup.beam
~/work/erlang/chapter12$ ls -1 src
db_server_app.erl
db_server_otp.erl
db_server_sup.erl

With the BEAM files and app files in the ebin directory, the application can now be tested with the Erlang shell.

Erlang R13B03 (erts-5.7.4)  [64-bit] [smp:8:2] [rq:8] [async-threads:0] [hipe] [kernel-poll:false]
Eshell V5.7.4  (abort with ^G)
1> code:add_path("/home/bryan/work/erlang/chapter12/ebin").
true
2> application:start(db_server).                               
ok
3> db_server_otp:write("Ted", "7").
ok
4> db_server_otp:read("Ted").
{ok,"7"}
5> application:stop(db_server).
ok

=INFO REPORT==== 3-Nov-2011::16:32:08 ===
    application: db_server
    exited: stopped
    type: temporary
6> whereis(db_server_sup).
undefined

Success! (Note: this was much easier than the first time I tried to get a Tomcat servlet working…)

Erlang Proramming Exercise 12-2: Supervising the Database Server

-module(db_server_sup).
-export([start/0,init/1]).
-behavior(supervisor).

start() ->
    supervisor:start_link({local, ?MODULE}, ?MODULE, []).

init(_Arguments) ->
    DbServerOtp = {db_server_otp,                %% Id
                   {db_server_otp, start, []},   %% child process
                   permanent,                    %% restart 
                   30000,                        %% shutdown (ms)
                   worker,                       %% type
                   [db_server_otp]},             %% required modules
    {ok, 
     {{one_for_all,           %% terminate all children and restart
       5,                     %% max of n restarts in MaxSeconds
       3600},                 %% MaxSeconds (s)
      [DbServerOtp]}}.        %% child process list

EP Exercise 12-1: Database Server Revisited

-module(db_server_otp).
-export([start/0,stop/0]).
-export([write/2,read/1,delete/1,match/1]).
-export([init/1,terminate/2,handle_cast/2,handle_call/3]).
-behavior(gen_server).

%%
%% Operations & Maintenance API
%%

start() ->
    gen_server:start({local, ?MODULE}, ?MODULE, [], []).


stop() ->
    gen_server:cast(?MODULE, stop).

%%
%% DB Server API
%%

write(Key, Data) ->
    gen_server:call(?MODULE, {write, Key, Data}).

read(Key) ->
    gen_server:call(?MODULE, {read, Key}).

delete(Key) ->
    gen_server:call(?MODULE, {delete, Key}).

match(Element) ->
    gen_server:call(?MODULE, {match, Element}).

%%
%% OTP gen_server Callback Functions
%%


init(_Arguments) ->
    {ok, []}.

terminate(_Reason, _LoopData) ->
    ok.

handle_cast(stop, LoopData) ->
    {stop, normal, LoopData}.

handle_call({write, Key, Data}, _From, LoopData) ->
    case lists:keymember(Key, 1, LoopData) of
        true ->
            NewLoopData = lists:keyreplace(Key, 1, LoopData, {Key, Data}),
            Reply = ok;
        false ->
            NewLoopData = LoopData ++ [{Key, Data}],
            Reply = ok
    end,
    {reply, Reply, NewLoopData};
handle_call({read, Key}, _From, LoopData) ->
    case lists:keymember(Key, 1, LoopData) of
        true ->
            {_, Value} = lists:keyfind(Key, 1, LoopData),
            Reply = {ok, Value};
        false ->
            Reply = {error, instance}
    end,
    {reply, Reply, LoopData};

handle_call({delete, Key}, _From, LoopData) ->
    case lists:keymember(Key, 1, LoopData) of
        true ->
            NewLoopData = lists:keydelete(Key, 1, LoopData),
            Reply = ok;
        false ->
            NewLoopData = LoopData,
            Reply = ok
    end,
    {reply, Reply, NewLoopData};

handle_call({match, Element}, _From, LoopData) ->
    Matches = lists:filter(
                fun({_ ,Value}) ->
                  if 
                     Value =:= Element -> true;
                     true -> false
                  end end, 
                LoopData),
   Reply = get_db_keys(Matches, []),
   {reply, Reply, LoopData}.
                                   

get_db_keys([], KeyList) -> KeyList;
get_db_keys(ListOfTuples, KeyList) ->
    [H|T] = ListOfTuples,
    {Key, _} = H,
    get_db_keys(T, KeyList ++ [Key]).

EP Exercise 5-1: A Database Server

-module(db_server).
-export([start/0,stop/0,upgrade/1,code_upgrade/0]).
-export([write/2,read/1,delete/1]).
-export([init/0,loop/1]).
-vsn(1.0).

start() ->
    register(deb_server, spawn(db_server, init, [])).

stop() ->
    db_server ! stop.

upgrade(Data) ->
    db_server ! {upgrade, Data}.

write(Key, Data) ->
    db_server ! {write, Key, Data}.

read(Key) ->
    db_server ! {read, self(), Key},
    receive Reply ->
             Reply end.

delete(Key) ->
    db_server ! {delete, Key}.

code_upgrade(Data) ->
    db_server ! {code_upgrade, Data}.

init() ->
    loop(db:new()).

loop(Db) ->
    receive
        {write, Key, Data} ->
            loop(db:write(Key, Data, Db));
        {read, Pid, Key} ->
            Pid ! db:read(Key, Db),
            loop(Db);
        {delete, Key} ->
            loop(db:delete(Key, Db));
        {upgrade, Data} ->
            NewDb = db:convert(Data, Db),
            db_server:loop(NewDb);
        code_upgrade ->
            loop(db:code_upgrade(Data));
        stop ->
            db:destroy(Db)
    end.

EP Exercise 4-2: The Process Ring

My solution for Exercise 4-2 in the book Erlang Programming

%%
%% Erlang Programming Exercise 4-2
%%

-module(ring).
-export([start/3,create_node/3,stop/0]).

create_node(M,N,Msg) ->
    io:format("Creating node ~w (~w)~n", [N,self()]),
    if  
        N-1 > 0  -> NextPid = spawn(ring,create_node,[M,N-1,Msg]);
        N-1 == 0 -> NextPid = head
    end,
    print_m_times(NextPid,N,Msg,M),
    loop(NextPid).

start(M, N, Msg) ->
    io:format("Spawning ~w nodes!~n",[N]),
    Pid = spawn(ring,create_node,[M,N,Msg]),
    register(head, Pid),
    ok. 

stop() ->
    head ! quit,
    ok. 

loop(NextPid) ->
    receive
        {print, Pid, NodeNum, MsgContents} when Pid /= self() ->  
            io:format("(~w) msg from node ~w: ~s~n",[self(),NodeNum,MsgContents]),
            NextPid ! {print, Pid, NodeNum, MsgContents},
            loop(NextPid);
        quit ->  
            NextPid ! quit,
            io:format("Destroying node (~w)~n", [self()]),
            true
    end.

print_m_times(_,_,_,0) ->
    true;
print_m_times(NextPid,FromNode,Msg,M_Times) ->
    NextPid ! {print, self(), FromNode, Msg},
    print_m_times(NextPid,FromNode,Msg,M_Times-1).

EP Exercise 4-1: An Echo Server

My solution for Exercise 4-1 in the book Erlang Programming.

%%
%% Erlang Programming Exercise 4-1
%%

-module(echo).
-export([start/0,print/1,stop/0,loop/0]).

start() ->
    %% function specified in spawn/3 must be exported !!
    Pid = spawn(echo, loop, []),
    register(echo_server, Pid),
    ok. 

stop() ->
    echo_server ! stop,
    ok. 

print(Term) ->
    echo_server ! {print, Term},
    ok. 

loop() ->
    receive
        {print, Msg} -> io:format("~s~n",[Msg]),
                        loop();
        stop -> true
    end.

RabbitMQ Install – painless and simple!

I always install Erlang from the source, so the first step was to tell RPM I don’t care about Erlang package dependencies and I just want it to install RabbitMQ.

sudo rpm –nodeps -i http://www.rabbitmq.com/releases/rabbitmq-server/v2.4.1/rabbitmq-server-2.4.1-1.suse.noarch.rpm

It installs silently, always the best way.  There only a few steps that were needed to setup RabbitMQ to run as a daemon.  First, I had to modify the startup script, /etc/init.d/rabbitmq-server, so that the path to the Erlang shell erl (/usr/local/bin) was in the PATH variable in the script.  I suspect if I had installed Erlang via a package this step would not be necessary.

The next step was to switch on RabbitMQ as a daemon and then start the server:

sudo /sbin/chkconfig rabbitmq-server on
sudo /sbin/service rabbitmq-server start

Just to be safe I checked the startup logs, /var/log/rabbitmq/startup_log and /var/log/rabbitmq/startup_err to make sure everything looked OK.  I also checked the status of the server using the rabbitmqctrl tool.

sudo /usr/sbin/rabbitmqctl status

On the terminal you get a list of tuples that contains various information about the server’s status.

Status of node rabbit@<some host> …
[{pid,3439},
{running_applications,[{rabbit,"RabbitMQ","2.4.1"},
{os_mon,"CPO  CXC 138 46","2.2.6"},
{sasl,"SASL  CXC 138 11","2.1.9.4"},
{mnesia,"MNESIA  CXC 138 12","4.4.19"},
{stdlib,"ERTS  CXC 138 10","1.17.4"},
{kernel,"ERTS  CXC 138 10","2.14.4"}]},
{nodes,[{disc,[rabbit@pt01]}]},
{running_nodes,[rabbit@pt01]}]
…done.

Easy!  My test python scripts had no trouble hooking up to RabbitMQ and exchanging messages.  This is so much easier than CouchDB!

Disappointment with CouchDB

I am looking for a no-sql database to simplify the management of some data I need for a project at work.  Tokyo Cabinet and MongoDB were recommended to me, but I decided I would give CouchDB a shot.  I read the Erlang mailing list regularly and CouchDB is often mentioned.  I also want to support software projects that use Erlang because I love working with Erlang and would love to help it spread.

Unfortunately I cannot recommend CouchDB.  Trying to install its dependencies were such a mess that I am no longer going to waste time with CouchDB.  Sorry guys, it looked really interesting, but I have work to do an other no-sql databases are much easier to setup.

Part of the issue is that I’m working on a Novell SLES 10 box, and as such it doesn’t get the most up-to-date packages. I don’t have the option of grabbing the latest version of Ubuntu or OpenSuSE due to audit department requirements.  I’m stuck with SLES 10 for now, if I’m lucky I will be able to upgrade the server to SLES 11 one day.

However, I was able to get Erlang installed without a hitch.  I have never once had an issue installing Erlang on a system: Solaris 10, Mac OS X, Windows, Ubuntu, and SLES.

sudo rpm -Uvh erlang-R13B01-13.1.x86_64.rpm   (from http://software.opensuse.org)

I went ahead and ran the CouchDB configure script just to see if all of the dependencies (libcurl, ICU, SpiderMonkey, OpenSSL) were optional.

checking for JS_NewContext in -lmozjs… no
checking for JS_NewContext in -ljs… no
checking for JS_NewContext in -ljs3250… no
checking for JS_NewContext in -ljs32… no
configure: error: Could not find the js library.

Is the Mozilla SpiderMonkey library installed?

Of course not, looks like I cannot skip SpiderMonkey.  And of course the Mozilla guys cannot make anything easy, I have to install Mercurial first just to get at the code.  Luckily there was a package for mercurial.  I think this server now has CVS, SVN, Bazaar, git and mercurial on it now.

sudo rpm -Uvh mercurial-1.8.3-45.2.x86_64.rpm

It took awhile for mecurial to clone the repistory, but it did grab everything.  I also had to install autoconf-2.13, and thankfully it also had a package for SLES 10.

hg clone http://hg.mozilla.org/mozilla-central/
cd mozilla-central/js/src
autoconf-2.13

I threw my hands up with the error that came up when I ran gmake as directed by the install.

gmake export
gmake[1]: Entering directory `/home/ptuser/nexttick/spidermonkey/mozilla-central/js/src’
gmake -C config/ nsinstall
gmake[2]: Entering directory `/home/ptuser/nexttick/spidermonkey/mozilla-central/js/src/config’
nsinstall.c
gcc -o host_nsinstall.o -c  -Wall -W -Wno-unused -Wpointer-arith -Wcast-align -W -pedantic -Wno-long-long -fno-strict-aliasing -pthread -pipe  -DNDEBUG -DTRIMMED -g -O3 -DXP_UNIX -O3  -DUNICODE -D_UNICODE  -I. -I. -I../dist/include -I../dist/include/nsprpub
gcc: no input files
gmake[2]: *** [host_nsinstall.o] Error 1
gmake[2]: Leaving directory `/home/ptuser/nexttick/spidermonkey/mozilla-central/js/src/config’
gmake[1]: *** [config/nsinstall] Error 2
gmake[1]: Leaving directory `/home/ptuser/nexttick/spidermonkey/mozilla-central/js/src’
gmake: *** [default] Error 2

I don’t fault the CouchDB guys for SpiderMonkey’s build system, but the amount of work required just to get the prerequisties for CouchDB makes it not worth my time.  In the amount of time I spent tracking down packages and trying to get everything, I could have just installed SQLlite and written some code.

I’m going to give MongoDB a shot now.

UPDATE: MongoDB also depends on SpiderMonkey, so it is out the window too.  I tried to install their 64-bit Linux binary, but I just got a floating-point exception.  Looks like I’ll just stick with Sqlite.  Besides, Sqlite3 is built into Python.

A Whoops Moment with Erlang

I”ll building my first non-trivial Erlang application, a server using the gen_tcp library, and I was left scratching my head for awhile with a problem today.  When I tried to connect to the server and send data with a really simple Python script, no data was being received by the server.  The incoming connection was handled by accept/1 properly, and the Python script didn’t seem to be reporting any errors.

I have to admit that I know next to nothing of Python, and I still consider myself a beginner in Erlang, so I was a bit concerned.  I wasn’t quit sure where the problem was being caused, so I switched the test client over to Erlang first.  I still could not receive any data at the server, so the problem seemed to be server-side.  Next I fired up the Erlang debugger to follow the execution of the server.  Sure enough, the server accepted the incoming connection and was blocking in the receive area.

The next step, which in hindsight I should have done first,  was to check all of the parameters to the listen/2.  Everything looked good, but I was unsure about the packet parameter and what it is used for.

gen_tcp:listen(
 PortNum,
 [list,
 {backlog, MaxCon},
 {active, once}, 
 {packet, 2},
 {reuseaddr, true}
])

Why was packet set to 2 again?  The listen/2 man page doesn’t mention anything about this parameter, but the inet:setopts man page says the following:

Defines the type of packets to use for a socket. The following values are valid:

raw | 0

No packaging is done.

1 | 2 | 4

Packets consist of a header specifying the number of bytes in the packet, followed by that number of bytes. The length of header can be one, two, or four bytes; the order of the bytes is big-endian. Each send operation will generate the header, and the header will be stripped off on each receive operation. 

 What I did not realize, however, is that this parameter is not simply internal to Erlang.  If the client and the server don’t have the same settings, no data will flow between the two.  I changed the packet value to 0 in the server, and after rerunning the test client script, sure enough, data flowed from the client to the server.

I really felt like an idiot for missing this.

Update on my Erlang experience

It has been awhile since I updated my blog.  I attribute it to work–the never ending stream of things to do kind of weights me down and I lose interest in all things technical from time to time.  I’m on a new project at work now, and I’m quite keen for it.  It’s still the same old low level C/Assembly firmware development, but I get to work with serial RapidIO technology and that has been quite interesting.  3.125 Gbps links!  I like this better than the former project working on MAC-layer software for a communication protocol.  I either want to be working at the PHY level directly accessing or working with the hardware, or I want to be up higher on the application level solving interesting problems.

Speaking of the application level, in late December I picked up my Erlang studies again, and I’m making good progress in understanding how to use it.  I’m implementing a simple chat server so I can wrap my mind around Erlang processes, file I/O and TCP sockets.  Having programmed a server in C/C++ before I can really appreciate how great Erlang is for network computing.  The code is so much more concise than the multithreaded C++ server I wrote in the past.  I’m not using the Erlang OTP libraries yet, I’m working with the gen_tcp library since it is familiar to BSD-style socket programming.  I’ve even dabled in a some Python scripts for testing the server.  Like how Perl has become my tool of choice for “glueing” things together, I can see myself using Python more often for testing in the future.  

Progress has been good, but I feel really shaky with strings at the moment.  Manipulating text is something any software has to do, but I find that the first thing that pops to my mind is the C++ STL string library, so I go searching for a similar library in the Erlang man pages.  I suppose that this must be expected–I just don’t know which libraries to use yet and it will take time to get the hang of them.  It took me awhile to master the use of the C++ STL string library and its algorithms, and I’m just going to have to figure out which Erlang libraries have the functions I need.  Perhaps I’ll draft a tutorial on Erlang string processing if I learn enough?

The good news is that my confidence with the language is increasing, tail recursion doesn’t seem that awkward, and immutable variables make complete sense now.  I won’t be using Erlang at work any time soon (not the right problem domain), but for personal enrichment and for learning new ways of doing things I’m quite pleased with Erlang. If I ever get into application network programming (or network services), I know what tool I’ll be choosing.