001package com.github.sarxos.webcam.ds.gstreamer; 002 003import java.awt.Dimension; 004import java.awt.image.BufferedImage; 005import java.awt.image.DataBufferInt; 006import java.io.File; 007import java.nio.IntBuffer; 008import java.util.ArrayList; 009import java.util.List; 010import java.util.concurrent.TimeUnit; 011import java.util.concurrent.atomic.AtomicBoolean; 012 013import org.bridj.Platform; 014import org.gstreamer.Caps; 015import org.gstreamer.Element; 016import org.gstreamer.ElementFactory; 017import org.gstreamer.Pad; 018import org.gstreamer.Pipeline; 019import org.gstreamer.State; 020import org.gstreamer.Structure; 021import org.gstreamer.elements.RGBDataSink; 022import org.slf4j.Logger; 023import org.slf4j.LoggerFactory; 024 025import com.github.sarxos.webcam.WebcamDevice; 026import com.github.sarxos.webcam.WebcamResolution; 027 028 029public class GStreamerDevice implements WebcamDevice, RGBDataSink.Listener, WebcamDevice.FPSSource { 030 031 /** 032 * Logger. 033 */ 034 private static final Logger LOG = LoggerFactory.getLogger(GStreamerDevice.class); 035 036 /** 037 * Limit the lateness of frames to no more than 20ms (half a frame at 25fps) 038 */ 039 private static final long LATENESS = 20; // ms 040 041 /** 042 * Video format to capture. 043 */ 044 private static final String FORMAT_MIME = "video/x-raw-yuv"; 045 046 /** 047 * All possible resolutions - populated while initialization phase. 048 */ 049 private Dimension[] resolutions = null; 050 051 /** 052 * Device name, immutable. Used only on Windows platform. 053 */ 054 private final String name; 055 /** 056 * Device name, immutable. Used only on Linux platform. 057 */ 058 private final File vfile; 059 060 /* gstreamer stuff */ 061 062 private Pipeline pipe = null; 063 private Element source = null; 064 private Element filter = null; 065 private RGBDataSink sink = null; 066 067 private Caps caps = null; 068 069 /* logic */ 070 071 private AtomicBoolean open = new AtomicBoolean(false); 072 private AtomicBoolean disposed = new AtomicBoolean(false); 073 private AtomicBoolean starting = new AtomicBoolean(false); 074 private AtomicBoolean initialized = new AtomicBoolean(false); 075 private Dimension resolution = WebcamResolution.VGA.getSize(); 076 private BufferedImage image = null; 077 078 /* used to calculate fps */ 079 080 private long t1 = -1; 081 private long t2 = -1; 082 083 private volatile double fps = 0; 084 085 /** 086 * Create GStreamer webcam device. 087 * 088 * @param name the name of webcam device 089 */ 090 protected GStreamerDevice(String name) { 091 this.name = name; 092 this.vfile = null; 093 } 094 095 protected GStreamerDevice(File vfile) { 096 this.name = null; 097 this.vfile = vfile; 098 } 099 100 /** 101 * Initialize webcam device. 102 */ 103 private synchronized void init() { 104 105 if (!initialized.compareAndSet(false, true)) { 106 return; 107 } 108 109 LOG.debug("GStreamer webcam device initialization"); 110 111 pipe = new Pipeline(name); 112 113 if (Platform.isWindows()) { 114 source = ElementFactory.make("dshowvideosrc", "source"); 115 source.set("device-name", name); 116 } else if (Platform.isLinux()) { 117 source = ElementFactory.make("v4l2src", "source"); 118 source.set("device", vfile.getAbsolutePath()); 119 } 120 121 sink = new RGBDataSink(name, this); 122 sink.setPassDirectBuffer(true); 123 sink.getSinkElement().setMaximumLateness(LATENESS, TimeUnit.MILLISECONDS); 124 sink.getSinkElement().setQOSEnabled(true); 125 126 filter = ElementFactory.make("capsfilter", "filter"); 127 128 if (Platform.isLinux()) { 129 pipe.addMany(source, filter, sink); 130 Element.linkMany(source, filter, sink); 131 pipe.setState(State.READY); 132 } 133 134 resolutions = parseResolutions(source.getPads().get(0)); 135 136 if (Platform.isLinux()) { 137 pipe.setState(State.NULL); 138 Element.unlinkMany(source, filter, sink); 139 pipe.removeMany(source, filter, sink); 140 } 141 } 142 143 /** 144 * Use GStreamer to get all possible resolutions. 145 * 146 * @param pad the pad to get resolutions from 147 * @return Array of resolutions supported by device connected with pad 148 */ 149 private static final Dimension[] parseResolutions(Pad pad) { 150 151 List<Dimension> dimensions = new ArrayList<Dimension>(); 152 153 Caps caps = pad.getCaps(); 154 155 Structure structure = null; 156 String mime = null; 157 158 int n = caps.size(); 159 int i = 0; 160 161 int w = -1; 162 int h = -1; 163 164 do { 165 166 structure = caps.getStructure(i++); 167 168 LOG.debug("Found format structure {}", structure); 169 170 mime = structure.getName(); 171 172 if (mime.equals(FORMAT_MIME)) { 173 if (Platform.isWindows()) { 174 w = structure.getRange("width").getMinInt(); 175 h = structure.getRange("height").getMinInt(); 176 dimensions.add(new Dimension(w, h)); 177 } else if (Platform.isLinux()) { 178 if ("YUY2".equals(structure.getFourccString("format"))) { 179 w = structure.getInteger("width"); 180 h = structure.getInteger("height"); 181 dimensions.add(new Dimension(w, h)); 182 } 183 } 184 } 185 186 } while (i < n); 187 188 return dimensions.toArray(new Dimension[dimensions.size()]); 189 } 190 191 @Override 192 public String getName() { 193 if (Platform.isWindows()) { 194 return name; 195 } else if (Platform.isLinux()) { 196 return vfile.getAbsolutePath(); 197 } else { 198 throw new RuntimeException("Platform not supported by GStreamer capture driver"); 199 } 200 } 201 202 @Override 203 public Dimension[] getResolutions() { 204 init(); 205 return resolutions; 206 } 207 208 @Override 209 public Dimension getResolution() { 210 return resolution; 211 } 212 213 @Override 214 public void setResolution(Dimension size) { 215 this.resolution = size; 216 } 217 218 @Override 219 public BufferedImage getImage() { 220 return image; 221 } 222 223 @Override 224 public void open() { 225 226 if (!open.compareAndSet(false, true)) { 227 return; 228 } 229 230 LOG.debug("Opening GStreamer device"); 231 232 init(); 233 234 starting.set(true); 235 236 Dimension size = getResolution(); 237 238 image = new BufferedImage(size.width, size.height, BufferedImage.TYPE_INT_RGB); 239 image.setAccelerationPriority(0); 240 image.flush(); 241 242 if (caps != null) { 243 caps.dispose(); 244 } 245 246 caps = Caps.fromString(String.format("%s,width=%d,height=%d", FORMAT_MIME, size.width, size.height)); 247 248 filter.setCaps(caps); 249 250 LOG.debug("Link elements"); 251 252 pipe.addMany(source, filter, sink); 253 Element.linkMany(source, filter, sink); 254 pipe.setState(State.PLAYING); 255 256 // wait max 20s for image to appear 257 synchronized (this) { 258 LOG.debug("Wait for device to be ready"); 259 try { 260 this.wait(20000); 261 } catch (InterruptedException e) { 262 return; 263 } 264 } 265 } 266 267 @Override 268 public void close() { 269 270 if (!open.compareAndSet(true, false)) { 271 return; 272 } 273 274 LOG.debug("Closing GStreamer device"); 275 276 image = null; 277 278 LOG.debug("Unlink elements"); 279 280 pipe.setState(State.NULL); 281 Element.unlinkMany(source, filter, sink); 282 pipe.removeMany(source, filter, sink); 283 } 284 285 @Override 286 public void dispose() { 287 288 if (!disposed.compareAndSet(false, true)) { 289 return; 290 } 291 292 LOG.debug("Disposing GStreamer device"); 293 294 close(); 295 296 filter.dispose(); 297 source.dispose(); 298 sink.dispose(); 299 pipe.dispose(); 300 caps.dispose(); 301 } 302 303 @Override 304 public boolean isOpen() { 305 return open.get(); 306 } 307 308 @Override 309 public void rgbFrame(boolean preroll, int width, int height, IntBuffer rgb) { 310 311 LOG.trace("New RGB frame"); 312 313 if (t1 == -1 || t2 == -1) { 314 t1 = System.currentTimeMillis(); 315 t2 = System.currentTimeMillis(); 316 } 317 318 BufferedImage tmp = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB); 319 tmp.setAccelerationPriority(0); 320 tmp.flush(); 321 322 rgb.get(((DataBufferInt) tmp.getRaster().getDataBuffer()).getData(), 0, width * height); 323 324 image = tmp; 325 326 if (starting.compareAndSet(true, false)) { 327 328 synchronized (this) { 329 this.notifyAll(); 330 } 331 332 LOG.debug("GStreamer device ready"); 333 } 334 335 t1 = t2; 336 t2 = System.currentTimeMillis(); 337 338 fps = (4 * fps + 1000 / (t2 - t1 + 1)) / 5; 339 } 340 341 @Override 342 public double getFPS() { 343 return fps; 344 } 345 346 public Pipeline getPipe() { 347 return pipe; 348 } 349 350 public Element getSource() { 351 return source; 352 } 353 354 public Element getFilter() { 355 return filter; 356 } 357 358 public RGBDataSink getSink() { 359 return sink; 360 } 361 362 public Caps getCaps() { 363 return caps; 364 } 365}