compile ofp over dpdk in vmware

1. get software

1
2
git libtool automake build-essential pkg-config libssl-dev doxygen libconfig-dev libnuma-dev libpcap-dev net-tools
ln -s /usr/bin/python3 /usr/bin/python

2. compile dpdk

1
git clone http://dpdk.org/git/dpdk-stable --branch 19.11 --depth 1 dpdk-19.11

run dpdk-19.11/usertools/dpdk-setup.sh
>>> select and run "x86_64-native-linuxapp-gcc"
>>> Exit Script

3. compile odp-dpdk

1
git clone https://github.com/OpenDataPlane/odp-dpdk.git

odp-dpdk/m4/odp_dpdk.m4

1
2
3
4
5
6
7
8
9
10
11
12
13
# _ODP_DPDK_LEGACY(PATH, ACTION-IF-FOUND, ACTION-IF-NOT-FOUND)
# ------------------------------------------------------------------------
# Locate DPDK installation
AC_DEFUN([_ODP_DPDK_LEGACY], [dnl
    DPDK_CFLAGS="-isystem $1/include"
    DPDK_LIB_PATH="$1/lib"
    DPDK_LDFLAGS="-L$DPDK_LIB_PATH"
    AS_IF([test -r "$DPDK_LIB_PATH"/libdpdk.so], [dnl
        DPDK_RPATH="-Wl,-rpath,$DPDK_LIB_PATH"
        DPDK_RPATH_LT="-R$DPDK_LIB_PATH"
        DPDK_SHARED=yes],
        [test ! -r "$DPDK_LIB_PATH"/libdpdk.a], [dnl
        AC_MSG_FAILURE([Could not find DPDK])])
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
#!/bin/sh
 
DPDK_PATH=/home/eu/work/dpdk/dpdk-19.11/x86_64-native-linuxapp-gcc
 
cd odp-dpdk
./bootstrap
#--enable-debug --enable-debug-print
#--without-openssl
./configure --with-dpdk-path=${DPDK_PATH} \
                --prefix=`pwd`/installed \
                --enable-debug=full \
                --enable-helper-debug \
                --enable-debug-print \
                --enable-helper-debug-print \
                --disable-shared
make install

4. compile ofp

1
git clone https://github.com/OpenFastPath/ofp.git
1
2
3
4
5
6
7
8
9
10
11
12
13
#!/bin/sh
 
ODP_DPDK_PATH=`pwd`/odp-dpdk/installed
 
cd ofp
./bootstrap
./configure --with-odp=${ODP_DPDK_PATH} \
        --with-odp-lib=odp-dpdk \
        --disable-shared \
        --enable-debug=yes \
        --enable-sp=no \
        --prefix=`pwd`/installed
make install

5. setup dpdk

1
2
3
4
cd dpdk-19.11/x86_64-native-linuxapp-gcc/kmod
modprobe uio
insmod igb_uio.ko
ifconfig ens33 down

run dpdk-19.11/usertools/dpdk-setup.sh
>>> select and run "Setup hugepage mappings for non-NUMA systems", enter 128
>>> select and run "Bind Ethernet/Baseband/Crypto device to IGB UIO module", will display like
0000:02:01.0 '82545EM Gigabit Ethernet Controller (Copper) 100f' if=ens33 drv=e1000 unused=igb_uio,vfio-pci
>>>>>> enter 02:01.0
>>> Exit Script

6. test

let the vmware guest machine use NAT.

use ipconfig check ip settings in host machine. for example, my ip for "VMware Network Adapter VMnet8" is 192.168.204.1.

ofp/example/webserver/ofp.cli

1
2
3
debug 0
loglevel set debug
ifconfig fp0 192.168.204.33/24

ofp/example/webserver/www/aa.txt

1
Hello, the world!

ofp/example/webserver/run.sh

1
2
export www_dir=`pwd`/www
./webserver -i 0 -f ofp.cli

using web browser access

1
http://192.168.204.33:2048/aa.txt

ofp segment fault的原因

1. 创建线程不要直接用pthread_create,而要用odph_odpthreads_create。
2. 每个线程函数的开始处都要调用ofp_init_local初始化。
3. 之前调用系统socket和epoll之类的函数都要替换成ofp定义的,包括宏和结构体。如

1
2
//	sock = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
	sock = ofp_socket(OFP_AF_INET, OFP_SOCK_STREAM, OFP_IPPROTO_TCP);

ofp中的慢平面

默认编译OpenFastPath时慢平面是启用的,慢平面的功能之一是通过netlink方式向内核查询路由表,这是用内核公开的接口,不需要另开发内核驱动。

慢平面的一个局限,比方电脑只有一张网卡,而这块网卡已被dpdk绑定,网络包就不会在内核中被解析,此时慢平面的路由功能也就无效,所以干脆通过

1
./configure ... --enable-sp=no

将慢平面功能去掉。

而此时我们跑在ofp上的应用就没有路由表了么?
不是的。首先应用的本机ip地址是通过

1
myApp -i 0 -f ofp.cli

中的ofp.cli配置的。如

1
2
3
debug 0
loglevel set debug
ifconfig fp0 192.168.56.33/24

中的192.168.56.33就是本机地址。
而通过arp协议获取其它机器ip与mac关系的路由表是由

1
	ofp_start_cli_thread(instance, app_init_params.linux_core_id, params.cli_file);

这个线程在dpdk绑定的网卡上进行解析维护。

一个很好的例子就是ofp/example/webserver。

ofp和dpdk都可完全编译成静态库运行,不要为了LD_PRELOAD的那一点便利一门心思去编译出动态库。在自已的源码中加一层宏翻译,静态库与epoll api兼容性也很好。

github上的odp与dpdk-odp的区别是前者现在只包含了odp-linux,一种对linux系统api的包装,后者是对dpdk的包装,所以高性能选后者。原话见
doc/users-guide/users-guide.adoc

refer to:
https://github.com/OpenFastPath/ofp
https://github.com/OpenDataPlane/odp-dpdk
https://my.oschina.net/u/4361425/blog/3269917

odp禁止ipsec

1
2
3
4
5
6
7
8
9
10
11
12
13
	odp_init_t init_param;
 
	odp_init_param_init(&init_param);
	init_param.not_used.feat.ipsec = 1;
	init_param.not_used.feat.crypto = 1;
	init_param.not_used.feat.compress = 1;
	init_param.not_used.feat.tm = 1;
 
	if (0 != odp_init_global(&instance, &init_param, NULL))
	{
		DBG(("init odp global failed.\n"));
		return 0;
	}

gtk user control自绘控件

ubuntu下gtk库是默认已安装的,开发部署很方便,以下是我写的自绘控件例子:

EuhatChildWnd.h

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
#pragma once
 
struct _cairo_surface;
struct _GtkWidget;
 
class EuhatChildWnd
{
public:
	EuhatChildWnd(int width, int height);
	~EuhatChildWnd();
 
	void clear();
	void drawBrush(struct _GtkWidget *widget, double x, double y);
 
	struct _GtkWidget *drawingArea_;
	struct _cairo_surface *surface_;
 
	double xFrom_;
	double yFrom_;
};

Read more

编译时打印宏变量内容

1
2
3
4
5
6
7
8
9
#define PRINT_MACRO_HELPER(_x) #_x
#define PRINT_MACRO(_x) #_x " = " PRINT_MACRO_HELPER(_x)
 
#define DEFINED_PI 3.14
#define DEFINED_NULL
#pragma message(PRINT_MACRO(DEFINED_PI))
#pragma message(PRINT_MACRO(DEFINED_NULL))
#pragma message(PRINT_MACRO(UNDEFINED))
//#error print stop here.

refer to: https://blog.csdn.net/xshbx/article/details/7981564

从cmdline解析出参数token

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
#include <string>
#include <vector>
#include <sstream>
 
using namespace std;
 
wstring copySpanStr(const wchar_t* start, const wchar_t* end)
{
	wchar_t* buf = (wchar_t*)malloc((end - start + 1) * sizeof(wchar_t));
	wchar_t* pTo = buf;
	for (const wchar_t* p = start; p != end; p++)
	{
		if (*p == L'\\' && p + 1 != end)
		{
			p++;
			*pTo++ = *p;
		}
		else
			*pTo++ = *p;
	}
	*pTo = 0;
	wstring out = buf;
	free(buf);
	return out;
}
 
int getParams(const wchar_t* str, vector<wstring>& out)
{
	int isInQMark = 0;
	int isInSpace = 1;
	const wchar_t* tokenStart = str;
	for (const wchar_t* p = str; ; p++)
	{
		if (*p == L'"')
		{
			if (!isInQMark)
			{
				isInQMark = 1;
				tokenStart = p + 1;
			}
			else
			{
				isInQMark = 0;
				out.push_back(copySpanStr(tokenStart, p));
			}
		}
		else if (*p == L'\\')
		{
			if (*(p + 1) != 0)
				p++;
		}
		else if (*p == L' ' || *p == 0)
		{
			if (!isInQMark)
			{
				if (!isInSpace)
				{
					isInSpace = 1;
					if (tokenStart != p)
					{
						out.push_back(copySpanStr(tokenStart, p));
						tokenStart = p;
					}
				}
				else
				{
 
				}
			}
		}
		else
		{
			if (!isInQMark)
			{
				if (!isInSpace)
				{
 
				}
				else
				{
					isInSpace = 0;
					tokenStart = p;
				}
			}
		}
		if (*p == 0)
			break;
	}
	return out.size();
}
 
wstring strReplaceCh2Str(const wchar_t* in, wchar_t fromCh, const wchar_t* toStr)
{
	const wchar_t* p = in;
	wstringstream ss;
	const wchar_t* pToStr;
	for (; *p != 0; p++)
	{
		if (*p == fromCh)
		{
			for (pToStr = toStr; *pToStr != 0; pToStr++)
			{
				ss << *pToStr;
			}
		}
		else
			ss << *p;
	}
	return ss.str();
}
 
int main()
{
	wstring path = strReplaceCh2Str(L"c:\\aa.txt", L'\\', L"\\\\");
	wstring msg = strReplaceCh2Str(L"\"优孩\"是EuhatExpert的中文名。", L'\"', L"\\\"");
 
	wstring cmd = L"EuhatExample.exe subCmd \"" + path + L"\" \"" + msg + L"\" 1234 5678";
 
	vector<wstring> params;
	getParams(cmd.c_str(), params);
	return 0;
}

dll中的全局类实例什么时候初始化

如果外部工程ExeB调了动态库DllA中的函数,当ExeB运行时,DllA中的全局类实例一定会初始化,但不一定非得在ExeB中调DllA中的函数后DllA中的全局类实例才会初始化,比如在DllA中代码如下

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
class DllA01()
{
public:
	DllA01()
	{
		MessageBoxA(NULL, "init", NULL, 0);
	}
	~DllA01()
	{
		MessageBoxA(NULL, "fini", NULL, 0);
	}
	void test(void)
	{
		MessageBoxA(NULL, "test", NULL, 0);
	}
};
__declspec(dllexport) DllA01 gA01;
extern "C" __declspec(dllexport) void dllA01Test(void)
{
	MessageBoxA(NULL, "dllA01Test", NULL, 0);
}

在ExeB中加入

1
2
3
4
#pragma comment(lib, "DllA.lib")
#pragma comment(linker, "/include:__imp__dllA01Test")
//#pragma comment(linker, "/include:__imp_?gA01@@3VDllA01@@A")
//#pragma comment(linker, "/ENTRY:foo")