TOE, Netmap, DPDK 그리고 클라우드

1.
TOE. TCP OFF Loading Engine의 약자입니다. Low Latency가 자본시장IT를 주도할 때 각광을 받았던 기술입니다. TOE는 비표준이기 하지만 매력적인 기술입니다. 더구나 가격도 많이 착해진 상태로 선택이 가능합니다. 그렇지만 지금처럼 가상화와 클라우드가 시대의 흐름을 이루면서 고민스러운 기술이 되었습니다. 2011년 ZeroAOS를 클라우드환경으로 운영하고자 할 때 이런 저런 조사와 고민을 하였습니다.

Latency와 가상화는 이율배반?

그리고 shared memory를 이용하여 해결하고자 했던 기술도 살폈습니다.

SR-IOV와 IPC

그리고 시간이 흘렀습니다. 클라우드와 관련한 고민은 진전시키지 못했습니다. 클라우드환경을 운영하고자 한 시도가 실패했습니다. 실패는 실패고 세상은 Docker와 같은 기술이 클라우드서버환경을 풍부하도록 변화하고 있습니다. Docker로 보는 클라우드 서버 운영의 미래는 이런 흐름을 보여줍니다.

</p> <p>클라우드와 TOE는 어떤 관계가 있을까요? IBM이 발표한 <a href="http://domino.research.ibm.com/library/cyberdig.nsf/papers/0929052195DD819C85257D2300681E7B/$File/rc25482.pdf" target="_blank">An Updated Performance Comparison of<br /> Virtual Machines and Linux Containers</a>을 읽어보면 docker이든 vm이든 IO상의 지연은 피할 수 없어 보입니다.</p> <iframe src="//docs.google.com/viewer?url=http%3A%2F%2Fwww.smallake.kr%2Fwp-content%2Fuploads%2F2015%2F03%2Frc25482.pdf&hl=ko&embedded=true" class="gde-frame" style="width:100%; height:500px; border: none;" scrolling="no">

Download (PDF, 377KB)

어떻게 해결해야 할까요? TOE와 비슷한 목적을 가지지만 다른 방법을 택하고 있는 기술중 Netmap이 있습니다. DIY Acceleration에서 소개하였고 NIC과 Userspace를 연결시키는 기술입니다만 RNIC에 의존하지 않는 방법입니다. 따라서 VM이나 Docker에 채택할 수 있습니다.

Netmap기술을 기반으로 하여 클라우드서버내에 가상스위치를 만들어 레이턴시를 해결하는 mSwitch기술도 내놓았습니다.

Download (PDF, 1.13MB)

이와 다른 기술이 인텔이 내놓은 DPDK입니다. 조사를 해보니 Openstack에 친화적인 기술입니다. DPDK, vswitch기술이 조합을 이룹니다.

트레이더가 클라우드환경을 구축하고 매매를 할 가능성이 많지 않습니다. 속도가 곧 수익이라고 하면 가상화와 같은 기술을 사용하지 않을 듯 합니다. 그렇지만 다른 요인으로 클라우드환경을 구축하면서 레이턴시를 고민할 경우가 있습니다. 매매의 경우 속도는 여전히 중요하기때문입니다. 이럴 때 무슨 선택을 할까요?

2.
RNIC이 처음 나왔던 때와 비교하면 지금은 무척 낮은 가격으로 이용할 수 있습니다. 혹시 TOE를 구매할 때 Non-TOE와 비교할 때 어느정도 Latency를 줄이는지를 비교해보신 적이 있으신가요? 솔직히 저도 없습니다. TOE Nic은 싼 제품이 아닙니다. 몇 십만원 하는 보급형이 아닌 일반형은 백만원이 넘습니다. 온 보드나 번들로 랜카드를 사용하던 분들은 쉽게 비용을 지불하기 힘듭니다. 그렇기 때문에 TOE를 이용하여 얻을 수 있는 이익을 측정하면 좋습니다.

Measuring latency in the Linux network stack between kernel and user space

위의 글을 이런 분들에게 아이디어를 제공합니다. Kernel이 nic에서 데이타를 받아서 User sapce로 보내는 시간을 측정하는 프로그램입니다.

/*
Copyright (c) 2008, Max Vilimpoc, http://vilimpoc.org/

All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

    * Redistributions of source code must retain the above
      copyright notice, this list of conditions and the following
      disclaimer.
    * Redistributions in binary form must reproduce the above
      copyright notice, this list of conditions and the following
      disclaimer in the documentation and/or other materials
      provided with the distribution.
    * Neither the name of the author nor the names of its
      contributors may be used to endorse or promote products
      derived from this software without specific prior written
      permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

#define _XOPEN_SOURCE 600

#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/ip.h>

#include <netinet/in.h>
#include <arpa/inet.h>

#include <sys/time.h>

#include <sys/ioctl.h>
#include <linux/if.h>

#include <unistd.h>
#include <stdio.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>

#include <signal.h>

#include <getopt.h>

typedef enum
{
    STRERROR_LEN           = 80,
    NUM_LATENCIES          = 32,
    DEFAULT_PORT           = 1025,
} CONSTS;

static int                 inSocket;
static struct sockaddr_in  inInAddr;
    
static int                 totalUsec;
static int                 totalPackets;

static int                 latencies[NUM_LATENCIES];
static int                 index;
static int                 rollingAverage;

static bool                keepRunning;



void Usage(const char * const progName)
{
    printf("\n");
    printf("Usage: %s [-i IP address] [-e ethx] [-p port]\n", progName);
    printf("\n");
    printf("       -h          Help.\n");
    printf("       -i          Specifies IP address of interface you want to listen to.\n");
    printf("       -e          Specifies the ethernet interface name you want to listen to.\n");
    printf("       -p          Specifies port number of packets you want to see.\n");
    printf("\n");
    printf("If no option is specified (they are all optional), then the program\n");
    printf("will listen on IPADDR_ANY (all interfaces), port 1025.");
    printf("\n");
    printf("\n");
}


static void printError(int errorCode, const char * const lastFunction)
{
    static char strError[STRERROR_LEN];
    memset(strError, 0, STRERROR_LEN);
    
    /* Old school. */
    perror(lastFunction);

    /* New school. */
    strerror_r(errorCode, strError, STRERROR_LEN);
    
    printf(strError);
    printf("\n");
}


void catchIntr(int signalNumber)
{
    /* Exit the main loop. */
    keepRunning = false;
}

int main(int argc, char **argv)
{
    int rc = 0;

    
    /* Defaults */
    inInAddr.sin_addr.s_addr    = INADDR_ANY;
    inInAddr.sin_port           = htons(DEFAULT_PORT);
    inInAddr.sin_family         = AF_INET;


    inSocket = socket(PF_INET, SOCK_DGRAM, 0);
    if (0 > inSocket)
    {
        printf("socket() call failed.\n");
        printError(inSocket, "socket");
        rc = -1;
        goto socket_failed;
    }
    
    /* Process cmdline opts. */
    char *shortOpts = "hi:e:p:";
    int   getoptRet;
    
    while(-1 != (getoptRet = getopt(argc, argv, shortOpts)))
    {
        switch(getoptRet)
        {
            case 'i':
                inInAddr.sin_addr.s_addr = inet_addr(optarg);
                break;
            case 'e':
                {
                struct ifreq fetchIfInfo;
                memset(&fetchIfInfo, 0, sizeof(struct ifreq));
                memcpy(fetchIfInfo.ifr_name, optarg, IFNAMSIZ - 1);
                
                /* Fetch the IP address to listen to based on interface name. */
                ioctl(inSocket, SIOCGIFADDR, &fetchIfInfo);
                
                struct sockaddr_in * const sockInfo = (struct sockaddr_in * const) &fetchIfInfo.ifr_addr;
                inInAddr.sin_addr.s_addr   = sockInfo->sin_addr.s_addr;
                }
                break;
            case 'p':
                inInAddr.sin_port        = htons(atoi(optarg));
                break;
            case 'h':
            case '?':
            default:
                Usage(argv[0]);
                goto normal_exit;
                break;
        }
    }
    
    
    printf("Listening to: %s:%d\n", inet_ntoa(inInAddr.sin_addr),
                                    ntohs(inInAddr.sin_port));

    int timestampOn = 1;
    rc = setsockopt(inSocket, SOL_SOCKET, SO_TIMESTAMP, (int *) &timestampOn, sizeof(timestampOn));
    if (0 > rc)
    {
        printf("setsockopt(SO_TIMESTAMP) failed.\n");
        printError(rc, "setsockopt");
        goto setsockopt_failed;
    }

    rc = bind(inSocket, (struct sockaddr *) &inInAddr, sizeof(struct sockaddr_in));
    if (0 > rc)
    {
        printf("UDP bind() failed.\n");
        printError(rc, "bind");
        goto bind_failed;
    }

    struct msghdr   msg;
    struct iovec    iov;
    char            pktbuf[2048];
    
    char            ctrl[CMSG_SPACE(sizeof(struct timeval))];
    struct cmsghdr *cmsg = (struct cmsghdr *) &ctrl;

    msg.msg_control      = (char *) ctrl;
    msg.msg_controllen   = sizeof(ctrl);    

    msg.msg_name         = &inInAddr;
    msg.msg_namelen      = sizeof(inInAddr);
    msg.msg_iov          = &iov;
    msg.msg_iovlen       = 1;
    iov.iov_base         = pktbuf;
    iov.iov_len          = sizeof(pktbuf);


    struct timeval  time_kernel, time_user;
    int             timediff;


    printf("Starting main loop.\n");
    
    for(keepRunning = true; keepRunning;)
    {
        rc = recvmsg(inSocket, &msg, 0);

        gettimeofday(&time_user, NULL);
        
        if (cmsg->cmsg_level == SOL_SOCKET &&
            cmsg->cmsg_type  == SCM_TIMESTAMP &&
            cmsg->cmsg_len   == CMSG_LEN(sizeof(time_kernel))) 
        {
            memcpy(&time_kernel, CMSG_DATA(cmsg), sizeof(time_kernel));
        }
        
        printf("\n");
        printf("time_kernel                  : %d.%06d\n", (int) time_kernel.tv_sec, 
                                                           (int) time_kernel.tv_usec);
        printf("time_user                    : %d.%06d\n", (int) time_user.tv_sec,   
                                                           (int) time_user.tv_usec);
        
        timediff      = (time_user.tv_sec - time_kernel.tv_sec) * 1000000 + 
                        (time_user.tv_usec - time_kernel.tv_usec);
        totalUsec    += timediff; 
        ++totalPackets;
        
        rollingAverage  += timediff;
        rollingAverage  -= latencies[index];
        latencies[index] = timediff;
        index = (index + 1) % NUM_LATENCIES;
        
        printf("Total Average                : %d/%d = %.2f us\n", totalUsec, 
                                                                   totalPackets, 
                                                                   (float) totalUsec / totalPackets);
        printf("Rolling Average (%d samples) : %.2f us\n", NUM_LATENCIES, 
                                                           (float) rollingAverage / NUM_LATENCIES);
    }

bind_failed:    
setsockopt_failed:
    close(inSocket);
socket_failed:
normal_exit:
    return rc;
}

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

Redistribution and use in source and binary forms, with or without

modification, are permitted provided that the following conditions

are met:

* Redistributions of source code must retain the above

disclaimer.

* Redistributions in binary form must reproduce the above

disclaimer in the documentation and/or other materials

provided with the distribution.

* Neither the name of the author nor the names of its

contributors may be used to endorse or promote products

derived from this software without specific prior written

permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS

"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT

LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR

A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR

CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,

EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,

PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR

PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF

LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING

NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS

SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#define _XOPEN_SOURCE 600

#include <sys/types.h>

#include <sys/socket.h>

#include <netinet/ip.h>

#include <netinet/in.h>

#include <arpa/inet.h>

#include <sys/time.h>

#include <sys/ioctl.h>

#include <linux/if.h>

#include <unistd.h>

#include <stdio.h>

#include <stdbool.h>

#include <stdint.h>

#include <stdlib.h>

#include <string.h>

#include <signal.h>

#include <getopt.h>

typedef enum

{

STRERROR_LEN = 80,

NUM_LATENCIES = 32,

DEFAULT_PORT = 1025,

} CONSTS;

static int inSocket;

static struct sockaddr_in inInAddr;

static int totalUsec;

static int totalPackets;

static int latencies[NUM_LATENCIES];

static int index;

static int rollingAverage;

static bool keepRunning;

void Usage(const char * const progName)

{

printf("\n");

printf("Usage: %s [-i IP address] [-e ethx] [-p port]\n", progName);

printf("\n");

printf(" -h Help.\n");

printf(" -i Specifies IP address of interface you want to listen to.\n");

printf(" -e Specifies the ethernet interface name you want to listen to.\n");

printf(" -p Specifies port number of packets you want to see.\n");

printf("\n");

printf("If no option is specified (they are all optional), then the program\n");

printf("will listen on IPADDR_ANY (all interfaces), port 1025.");

printf("\n");

}

static void printError(int errorCode, const char * const lastFunction)

{

static char strError[STRERROR_LEN];

memset(strError, 0, STRERROR_LEN);

/* Old school. */

perror(lastFunction);

/* New school. */

strerror_r(errorCode, strError, STRERROR_LEN);

printf(strError);

printf("\n");

}

void catchIntr(int signalNumber)

{

/* Exit the main loop. */

keepRunning = false;

}

int main(int argc, char **argv)

{

int rc = 0;

/* Defaults */

inInAddr.sin_addr.s_addr = INADDR_ANY;

inInAddr.sin_port = htons(DEFAULT_PORT);

inInAddr.sin_family = AF_INET;

inSocket = socket(PF_INET, SOCK_DGRAM, 0);

if (0 > inSocket)

{

printf("socket() call failed.\n");

printError(inSocket, "socket");

rc = -1;

goto socket_failed;

}

/* Process cmdline opts. */

char *shortOpts = "hi:e:p:";

int getoptRet;

while(-1 != (getoptRet = getopt(argc, argv, shortOpts)))

{

switch(getoptRet)

{

case 'i':

inInAddr.sin_addr.s_addr = inet_addr(optarg);

break;

case 'e':

{

struct ifreq fetchIfInfo;

memset(&fetchIfInfo, 0, sizeof(struct ifreq));

memcpy(fetchIfInfo.ifr_name, optarg, IFNAMSIZ - 1);

/* Fetch the IP address to listen to based on interface name. */

ioctl(inSocket, SIOCGIFADDR, &fetchIfInfo);

struct sockaddr_in * const sockInfo = (struct sockaddr_in * const) &fetchIfInfo.ifr_addr;

inInAddr.sin_addr.s_addr = sockInfo->sin_addr.s_addr;

}

break;

case 'p':

inInAddr.sin_port = htons(atoi(optarg));

break;

case 'h':

case '?':

default:

Usage(argv[0]);

goto normal_exit;

break;

}

printf("Listening to: %s:%d\n", inet_ntoa(inInAddr.sin_addr),

ntohs(inInAddr.sin_port));

int timestampOn = 1;

rc = setsockopt(inSocket, SOL_SOCKET, SO_TIMESTAMP, (int *) &timestampOn, sizeof(timestampOn));

if (0 > rc)

{

printf("setsockopt(SO_TIMESTAMP) failed.\n");

printError(rc, "setsockopt");

goto setsockopt_failed;

}

rc = bind(inSocket, (struct sockaddr *) &inInAddr, sizeof(struct sockaddr_in));

if (0 > rc)

{

printf("UDP bind() failed.\n");

printError(rc, "bind");

goto bind_failed;

}

struct msghdr msg;

struct iovec iov;

char pktbuf[2048];

char ctrl[CMSG_SPACE(sizeof(struct timeval))];

struct cmsghdr *cmsg = (struct cmsghdr *) &ctrl;

msg.msg_control = (char *) ctrl;

msg.msg_controllen = sizeof(ctrl);

msg.msg_name = &inInAddr;

msg.msg_namelen = sizeof(inInAddr);

msg.msg_iov = &iov;

msg.msg_iovlen = 1;

iov.iov_base = pktbuf;

iov.iov_len = sizeof(pktbuf);

struct timeval time_kernel, time_user;

int timediff;

printf("Starting main loop.\n");

for(keepRunning = true; keepRunning;)

{

rc = recvmsg(inSocket, &msg, 0);

gettimeofday(&time_user, NULL);

if (cmsg->cmsg_level == SOL_SOCKET &&

cmsg->cmsg_type == SCM_TIMESTAMP &&

cmsg->cmsg_len == CMSG_LEN(sizeof(time_kernel)))

{

memcpy(&time_kernel, CMSG_DATA(cmsg), sizeof(time_kernel));

}

printf("\n");

printf("time_kernel : %d.%06d\n", (int) time_kernel.tv_sec,

(int) time_kernel.tv_usec);

printf("time_user : %d.%06d\n", (int) time_user.tv_sec,

(int) time_user.tv_usec);

timediff = (time_user.tv_sec - time_kernel.tv_sec) * 1000000 +

(time_user.tv_usec - time_kernel.tv_usec);

totalUsec += timediff;

++totalPackets;

rollingAverage += timediff;

rollingAverage -= latencies[index];

latencies[index] = timediff;

index = (index + 1) % NUM_LATENCIES;

printf("Total Average : %d/%d = %.2f us\n", totalUsec,

totalPackets,

(float) totalUsec / totalPackets);

printf("Rolling Average (%d samples) : %.2f us\n", NUM_LATENCIES,

(float) rollingAverage / NUM_LATENCIES);

}

bind_failed:

setsockopt_failed:

close(inSocket);

socket_failed:

normal_exit:

return rc;

}

이 글 공유하기:

Leave a Comment 응답 취소